Honor the new omit_dir_times var.
[rsync/rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int inplace;
40 extern int make_backups;
41 extern int csum_length;
42 extern int ignore_times;
43 extern int size_only;
44 extern OFF_T max_size;
45 extern int io_timeout;
46 extern int protocol_version;
47 extern int always_checksum;
48 extern char *partial_dir;
49 extern char *basis_dir[];
50 extern int compare_dest;
51 extern int link_dest;
52 extern int whole_file;
53 extern int local_server;
54 extern int list_only;
55 extern int read_batch;
56 extern int only_existing;
57 extern int orig_umask;
58 extern int safe_symlinks;
59 extern long block_size; /* "long" because popt can't set an int32. */
60
61 extern struct exclude_list_struct server_exclude_list;
62
63 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
64 {
65         if (preserve_perms
66          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
67                 return 0;
68
69         if (am_root && preserve_uid && st->st_uid != file->uid)
70                 return 0;
71
72         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
73                 return 0;
74
75         return 1;
76 }
77
78 /* Perform our quick-check heuristic for determining if a file is unchanged. */
79 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
80 {
81         if (st->st_size != file->length)
82                 return 0;
83
84         /* if always checksum is set then we use the checksum instead
85            of the file time to determine whether to sync */
86         if (always_checksum && S_ISREG(st->st_mode)) {
87                 char sum[MD4_SUM_LENGTH];
88                 file_checksum(fn, sum, st->st_size);
89                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
90                                                         : MD4_SUM_LENGTH) == 0;
91         }
92
93         if (size_only)
94                 return 1;
95
96         if (ignore_times)
97                 return 0;
98
99         return cmp_modtime(st->st_mtime, file->modtime) == 0;
100 }
101
102
103 /*
104  * set (initialize) the size entries in the per-file sum_struct
105  * calculating dynamic block and checksum sizes.
106  *
107  * This is only called from generate_and_send_sums() but is a separate
108  * function to encapsulate the logic.
109  *
110  * The block size is a rounded square root of file length.
111  *
112  * The checksum size is determined according to:
113  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
114  * provided by Donovan Baarda which gives a probability of rsync
115  * algorithm corrupting data and falling back using the whole md4
116  * checksums.
117  *
118  * This might be made one of several selectable heuristics.
119  */
120 static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
121 {
122         int32 blength;
123         int s2length;
124
125         if (block_size)
126                 blength = block_size;
127         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
128                 blength = BLOCK_SIZE;
129         else {
130                 int32 c;
131                 int64 l;
132                 int cnt;
133                 for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
134                 if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
135                         blength = MAX_BLOCK_SIZE;
136                 else {
137                     blength = 0;
138                     do {
139                             blength |= c;
140                             if (len < (int64)blength * blength)
141                                     blength &= ~c;
142                             c >>= 1;
143                     } while (c >= 8);   /* round to multiple of 8 */
144                     blength = MAX(blength, BLOCK_SIZE);
145                 }
146         }
147
148         if (protocol_version < 27) {
149                 s2length = csum_length;
150         } else if (csum_length == SUM_LENGTH) {
151                 s2length = SUM_LENGTH;
152         } else {
153                 int32 c;
154                 int64 l;
155                 int b = BLOCKSUM_BIAS;
156                 for (l = len; l >>= 1; b += 2) {}
157                 for (c = blength; c >>= 1 && b; b--) {}
158                 /* add a bit, subtract rollsum, round up. */
159                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
160                 s2length = MAX(s2length, csum_length);
161                 s2length = MIN(s2length, SUM_LENGTH);
162         }
163
164         sum->flength    = len;
165         sum->blength    = blength;
166         sum->s2length   = s2length;
167         sum->count      = (len + (blength - 1)) / blength;
168         sum->remainder  = (len % blength);
169
170         if (sum->count && verbose > 2) {
171                 rprintf(FINFO,
172                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
173                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
174                         sum->s2length, (double)sum->flength);
175         }
176 }
177
178
179 /*
180  * Generate and send a stream of signatures/checksums that describe a buffer
181  *
182  * Generate approximately one checksum every block_len bytes.
183  */
184 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
185 {
186         size_t i;
187         struct map_struct *mapbuf;
188         struct sum_struct sum;
189         OFF_T offset = 0;
190
191         sum_sizes_sqroot(&sum, len);
192
193         if (len > 0)
194                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
195         else
196                 mapbuf = NULL;
197
198         write_sum_head(f_out, &sum);
199
200         for (i = 0; i < sum.count; i++) {
201                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
202                 char *map = map_ptr(mapbuf, offset, n1);
203                 uint32 sum1 = get_checksum1(map, n1);
204                 char sum2[SUM_LENGTH];
205
206                 if (f_copy >= 0)
207                         full_write(f_copy, map, n1);
208
209                 get_checksum2(map, n1, sum2);
210
211                 if (verbose > 3) {
212                         rprintf(FINFO,
213                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
214                                 (double)i, (double)offset, (long)n1,
215                                 (unsigned long)sum1);
216                 }
217                 write_int(f_out, sum1);
218                 write_buf(f_out, sum2, sum.s2length);
219                 len -= n1;
220                 offset += n1;
221         }
222
223         if (mapbuf)
224                 unmap_file(mapbuf);
225 }
226
227
228 /*
229  * Acts on file number @p i from @p flist, whose name is @p fname.
230  *
231  * First fixes up permissions, then generates checksums for the file.
232  *
233  * @note This comment was added later by mbp who was trying to work it
234  * out.  It might be wrong.
235  */
236 static void recv_generator(char *fname, struct file_struct *file, int i,
237                            int f_out, int f_out_name)
238 {
239         int fd = -1, f_copy = -1;
240         STRUCT_STAT st, partial_st;
241         struct file_struct *back_file = NULL;
242         int statret, stat_errno;
243         char *fnamecmp, *partialptr, *backupptr = NULL;
244         char fnamecmpbuf[MAXPATHLEN];
245         uchar fnamecmp_type;
246
247         if (list_only)
248                 return;
249
250         if (verbose > 2)
251                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
252
253         if (server_exclude_list.head
254             && check_exclude(&server_exclude_list, fname,
255                              S_ISDIR(file->mode)) < 0) {
256                 if (verbose) {
257                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
258                                 safe_fname(fname));
259                 }
260                 return;
261         }
262
263         if (dry_run > 1) {
264                 statret = -1;
265                 stat_errno = ENOENT;
266         } else {
267                 statret = link_stat(fname, &st,
268                                     keep_dirlinks && S_ISDIR(file->mode));
269                 stat_errno = errno;
270         }
271
272         if (only_existing && statret == -1 && stat_errno == ENOENT) {
273                 /* we only want to update existing files */
274                 if (verbose > 1) {
275                         rprintf(FINFO, "not creating new file \"%s\"\n",
276                                 safe_fname(fname));
277                 }
278                 return;
279         }
280
281         if (statret == 0 && !preserve_perms
282             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
283                 /* if the file exists already and we aren't perserving
284                  * permissions then act as though the remote end sent
285                  * us the file permissions we already have */
286                 file->mode = (file->mode & ~CHMOD_BITS)
287                            | (st.st_mode & CHMOD_BITS);
288         }
289
290         if (S_ISDIR(file->mode)) {
291                 /* The file to be received is a directory, so we need
292                  * to prepare appropriately.  If there is already a
293                  * file of that name and it is *not* a directory, then
294                  * we need to delete it.  If it doesn't exist, then
295                  * (perhaps recursively) create it. */
296                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
297                         delete_file(fname, DEL_TERSE);
298                         statret = -1;
299                 }
300                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
301                         if (!relative_paths || errno != ENOENT
302                             || create_directory_path(fname, orig_umask) < 0
303                             || do_mkdir(fname, file->mode) < 0) {
304                                 rsyserr(FERROR, errno,
305                                         "recv_generator: mkdir %s failed",
306                                         full_fname(fname));
307                         }
308                 }
309                 /* f_out is set to -1 when doing final directory-permission
310                  * and modification-time repair. */
311                 if (set_perms(fname, file, statret ? NULL : &st, 0)
312                     && verbose && f_out != -1)
313                         rprintf(FINFO, "%s/\n", safe_fname(fname));
314                 return;
315         } else if (max_size && file->length > max_size) {
316                 if (verbose > 1)
317                         rprintf(FINFO, "%s is over max-size\n", fname);
318                 return;
319         }
320
321         if (preserve_links && S_ISLNK(file->mode)) {
322 #if SUPPORT_LINKS
323                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
324                         if (verbose) {
325                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
326                                         full_fname(fname), file->u.link);
327                         }
328                         return;
329                 }
330                 if (statret == 0) {
331                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
332                         char lnk[MAXPATHLEN];
333                         int len;
334
335                         if (!dflag
336                             && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
337                                 lnk[len] = 0;
338                                 /* A link already pointing to the
339                                  * right place -- no further action
340                                  * required. */
341                                 if (strcmp(lnk, file->u.link) == 0) {
342                                         set_perms(fname, file, &st,
343                                                   PERMS_REPORT);
344                                         return;
345                                 }
346                         }
347                         /* Not the right symlink (or not a symlink), so
348                          * delete it. */
349                         delete_file(fname, dflag | DEL_TERSE);
350                 }
351                 if (do_symlink(file->u.link,fname) != 0) {
352                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
353                                 full_fname(fname), safe_fname(file->u.link));
354                 } else {
355                         set_perms(fname,file,NULL,0);
356                         if (verbose) {
357                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
358                                         safe_fname(file->u.link));
359                         }
360                 }
361 #endif
362                 return;
363         }
364
365         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
366                 if (statret != 0 ||
367                     st.st_mode != file->mode ||
368                     st.st_rdev != file->u.rdev) {
369                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
370                         delete_file(fname, dflag | DEL_TERSE);
371                         if (verbose > 2) {
372                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
373                                         safe_fname(fname),
374                                         (int)file->mode, (int)file->u.rdev);
375                         }
376                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
377                                 rsyserr(FERROR, errno, "mknod %s failed",
378                                         full_fname(fname));
379                         } else {
380                                 set_perms(fname,file,NULL,0);
381                                 if (verbose) {
382                                         rprintf(FINFO, "%s\n",
383                                                 safe_fname(fname));
384                                 }
385                         }
386                 } else {
387                         set_perms(fname, file, &st, PERMS_REPORT);
388                 }
389                 return;
390         }
391
392         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
393                 return;
394
395         if (!S_ISREG(file->mode)) {
396                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
397                         safe_fname(fname));
398                 return;
399         }
400
401         fnamecmp = fname;
402         fnamecmp_type = FNAMECMP_FNAME;
403
404         if (statret == -1 && basis_dir[0] != NULL) {
405                 int fallback_match = -1;
406                 int match_level = 0;
407                 int i = 0;
408                 do {
409                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
410                                  basis_dir[i], fname);
411                         if (link_stat(fnamecmpbuf, &st, 0) == 0
412                             && S_ISREG(st.st_mode)) {
413                                 statret = 0;
414                                 if (link_dest) {
415                                         if (!match_level) {
416                                                 fallback_match = i;
417                                                 match_level = 1;
418                                         } else if (match_level == 2
419                                             && !unchanged_attrs(file, &st))
420                                                 continue;
421                                         if (!unchanged_file(fnamecmpbuf, file, &st))
422                                                 continue;
423                                         fallback_match = i;
424                                         match_level = 2;
425                                         if (!unchanged_attrs(file, &st))
426                                                 continue;
427                                 }
428                                 match_level = 3;
429                                 break;
430                         }
431                 } while (basis_dir[++i] != NULL);
432                 if (statret == 0) {
433                         if (match_level < 3) {
434                                 i = fallback_match;
435                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
436                                          basis_dir[i], fname);
437                         }
438 #if HAVE_LINK
439                         if (link_dest && match_level == 3 && !dry_run) {
440                                 if (do_link(fnamecmpbuf, fname) < 0) {
441                                         if (verbose) {
442                                                 rsyserr(FINFO, errno,
443                                                         "link %s => %s",
444                                                         fnamecmpbuf,
445                                                         safe_fname(fname));
446                                         }
447                                         fnamecmp = fnamecmpbuf;
448                                         fnamecmp_type = i;
449                                 }
450                         } else
451 #endif
452                         {
453                                 fnamecmp = fnamecmpbuf;
454                                 fnamecmp_type = i;
455                         }
456                 }
457         }
458
459         if (statret == 0 && !S_ISREG(st.st_mode)) {
460                 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
461                 if (delete_file(fname, dflag | DEL_TERSE) != 0)
462                         return;
463                 statret = -1;
464                 stat_errno = ENOENT;
465         }
466
467         if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
468             && link_stat(partialptr, &partial_st, 0) == 0
469             && S_ISREG(partial_st.st_mode)) {
470                 if (statret == -1)
471                         goto prepare_to_open;
472         } else
473                 partialptr = NULL;
474
475         if (statret == -1) {
476                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
477                         return;
478                 if (stat_errno == ENOENT)
479                         goto notify_others;
480                 if (verbose > 1) {
481                         rsyserr(FERROR, stat_errno,
482                                 "recv_generator: failed to stat %s",
483                                 full_fname(fname));
484                 }
485                 return;
486         }
487
488         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
489                 if (verbose > 1)
490                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
491                 return;
492         }
493
494         if (update_only && fnamecmp_type == FNAMECMP_FNAME
495             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
496                 if (verbose > 1)
497                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
498                 return;
499         }
500
501         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
502                 ;
503         else if (unchanged_file(fnamecmp, file, &st)) {
504                 if (fnamecmp_type == FNAMECMP_FNAME)
505                         set_perms(fname, file, &st, PERMS_REPORT);
506                 return;
507         }
508
509 prepare_to_open:
510         if (partialptr) {
511                 st = partial_st;
512                 fnamecmp = partialptr;
513                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
514                 statret = 0;
515         }
516
517         if (dry_run || whole_file > 0) {
518                 statret = -1;
519                 goto notify_others;
520         }
521         if (read_batch)
522                 goto notify_others;
523
524         /* open the file */
525         fd = do_open(fnamecmp, O_RDONLY, 0);
526
527         if (fd == -1) {
528                 rsyserr(FERROR, errno, "failed to open %s, continuing",
529                         full_fname(fnamecmp));
530             pretend_missing:
531                 /* pretend the file didn't exist */
532                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
533                         return;
534                 statret = -1;
535                 goto notify_others;
536         }
537
538         if (inplace && make_backups) {
539                 if (!(backupptr = get_backup_name(fname))) {
540                         close(fd);
541                         return;
542                 }
543                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
544                         close(fd);
545                         goto pretend_missing;
546                 }
547                 if (robust_unlink(backupptr) && errno != ENOENT) {
548                         rsyserr(FERROR, errno, "unlink %s",
549                                 full_fname(backupptr));
550                         free(back_file);
551                         close(fd);
552                         return;
553                 }
554                 if ((f_copy = do_open(backupptr,
555                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
556                         rsyserr(FERROR, errno, "open %s",
557                                 full_fname(backupptr));
558                         free(back_file);
559                         close(fd);
560                         return;
561                 }
562                 fnamecmp_type = FNAMECMP_BACKUP;
563         }
564
565         if (verbose > 3) {
566                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
567                         safe_fname(fnamecmp), (double)st.st_size);
568         }
569
570         if (verbose > 2)
571                 rprintf(FINFO, "generating and sending sums for %d\n", i);
572
573 notify_others:
574         write_int(f_out, i);
575         if (protocol_version >= 29 && inplace && !read_batch)
576                 write_byte(f_out, fnamecmp_type);
577         if (f_out_name >= 0)
578                 write_byte(f_out_name, fnamecmp_type);
579
580         if (dry_run || read_batch)
581                 return;
582
583         if (statret == 0) {
584                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
585
586                 if (f_copy >= 0) {
587                         close(f_copy);
588                         set_perms(backupptr, back_file, NULL, 0);
589                         if (verbose > 1) {
590                                 rprintf(FINFO, "backed up %s to %s\n",
591                                         fname, backupptr);
592                         }
593                         free(back_file);
594                 }
595
596                 close(fd);
597         } else
598                 write_sum_head(f_out, NULL);
599 }
600
601
602 void generate_files(int f_out, struct file_list *flist, char *local_name,
603                     int f_out_name)
604 {
605         int i;
606         int phase = 0;
607         char fbuf[MAXPATHLEN];
608
609         if (verbose > 2) {
610                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
611                         (long)getpid(), flist->count);
612         }
613
614         if (verbose >= 2) {
615                 rprintf(FINFO,
616                         whole_file > 0
617                         ? "delta-transmission disabled for local transfer or --whole-file\n"
618                         : "delta transmission enabled\n");
619         }
620
621         /* we expect to just sit around now, so don't exit on a
622            timeout. If we really get a timeout then the other process should
623            exit */
624         io_timeout = 0;
625
626         for (i = 0; i < flist->count; i++) {
627                 struct file_struct *file = flist->files[i];
628                 struct file_struct copy;
629
630                 if (!file->basename)
631                         continue;
632                 /* we need to ensure that any directories we create have writeable
633                    permissions initially so that we can create the files within
634                    them. This is then fixed after the files are transferred */
635                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
636                         copy = *file;
637                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
638                          * handling of permissions is strange? */
639                         copy.mode |= S_IWUSR; /* user write */
640                         file = &copy;
641                 }
642
643                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
644                                file, i, f_out, f_out_name);
645         }
646
647         phase++;
648         csum_length = SUM_LENGTH;
649         ignore_times = 1;
650
651         if (verbose > 2)
652                 rprintf(FINFO,"generate_files phase=%d\n",phase);
653
654         write_int(f_out, -1);
655
656         /* files can cycle through the system more than once
657          * to catch initial checksum errors */
658         while ((i = get_redo_num()) != -1) {
659                 struct file_struct *file = flist->files[i];
660                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
661                                file, i, f_out, f_out_name);
662         }
663
664         phase++;
665         if (verbose > 2)
666                 rprintf(FINFO,"generate_files phase=%d\n",phase);
667
668         write_int(f_out, -1);
669
670         /* Read post-redo-phase MSG_DONE and any prior messages. */
671         get_redo_num();
672
673         if (preserve_hard_links)
674                 do_hard_links();
675
676         /* now we need to fix any directory permissions that were
677          * modified during the transfer */
678         for (i = 0; i < flist->count; i++) {
679                 struct file_struct *file = flist->files[i];
680                 if (!file->basename || !S_ISDIR(file->mode))
681                         continue;
682                 recv_generator(local_name ? local_name : f_name(file),
683                                file, i, -1, -1);
684         }
685
686         if (verbose > 2)
687                 rprintf(FINFO,"generate_files finished\n");
688 }