Document --delete-before and --delete's new "during" behavior.
[rsync/rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int delete_during;
38 extern int update_only;
39 extern int opt_ignore_existing;
40 extern int inplace;
41 extern int make_backups;
42 extern int csum_length;
43 extern int ignore_times;
44 extern int size_only;
45 extern OFF_T max_size;
46 extern int io_timeout;
47 extern int protocol_version;
48 extern int always_checksum;
49 extern char *partial_dir;
50 extern char *basis_dir[];
51 extern int compare_dest;
52 extern int link_dest;
53 extern int whole_file;
54 extern int local_server;
55 extern int list_only;
56 extern int read_batch;
57 extern int only_existing;
58 extern int orig_umask;
59 extern int safe_symlinks;
60 extern long block_size; /* "long" because popt can't set an int32. */
61
62 extern struct exclude_list_struct server_exclude_list;
63
64 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
65 {
66         if (preserve_perms
67          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
68                 return 0;
69
70         if (am_root && preserve_uid && st->st_uid != file->uid)
71                 return 0;
72
73         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
74                 return 0;
75
76         return 1;
77 }
78
79 /* Perform our quick-check heuristic for determining if a file is unchanged. */
80 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
81 {
82         if (st->st_size != file->length)
83                 return 0;
84
85         /* if always checksum is set then we use the checksum instead
86            of the file time to determine whether to sync */
87         if (always_checksum && S_ISREG(st->st_mode)) {
88                 char sum[MD4_SUM_LENGTH];
89                 file_checksum(fn, sum, st->st_size);
90                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
91                                                         : MD4_SUM_LENGTH) == 0;
92         }
93
94         if (size_only)
95                 return 1;
96
97         if (ignore_times)
98                 return 0;
99
100         return cmp_modtime(st->st_mtime, file->modtime) == 0;
101 }
102
103
104 /*
105  * set (initialize) the size entries in the per-file sum_struct
106  * calculating dynamic block and checksum sizes.
107  *
108  * This is only called from generate_and_send_sums() but is a separate
109  * function to encapsulate the logic.
110  *
111  * The block size is a rounded square root of file length.
112  *
113  * The checksum size is determined according to:
114  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
115  * provided by Donovan Baarda which gives a probability of rsync
116  * algorithm corrupting data and falling back using the whole md4
117  * checksums.
118  *
119  * This might be made one of several selectable heuristics.
120  */
121 static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
122 {
123         int32 blength;
124         int s2length;
125
126         if (block_size)
127                 blength = block_size;
128         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
129                 blength = BLOCK_SIZE;
130         else {
131                 int32 c;
132                 int64 l;
133                 int cnt;
134                 for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
135                 if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
136                         blength = MAX_BLOCK_SIZE;
137                 else {
138                     blength = 0;
139                     do {
140                             blength |= c;
141                             if (len < (int64)blength * blength)
142                                     blength &= ~c;
143                             c >>= 1;
144                     } while (c >= 8);   /* round to multiple of 8 */
145                     blength = MAX(blength, BLOCK_SIZE);
146                 }
147         }
148
149         if (protocol_version < 27) {
150                 s2length = csum_length;
151         } else if (csum_length == SUM_LENGTH) {
152                 s2length = SUM_LENGTH;
153         } else {
154                 int32 c;
155                 int64 l;
156                 int b = BLOCKSUM_BIAS;
157                 for (l = len; l >>= 1; b += 2) {}
158                 for (c = blength; c >>= 1 && b; b--) {}
159                 /* add a bit, subtract rollsum, round up. */
160                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
161                 s2length = MAX(s2length, csum_length);
162                 s2length = MIN(s2length, SUM_LENGTH);
163         }
164
165         sum->flength    = len;
166         sum->blength    = blength;
167         sum->s2length   = s2length;
168         sum->count      = (len + (blength - 1)) / blength;
169         sum->remainder  = (len % blength);
170
171         if (sum->count && verbose > 2) {
172                 rprintf(FINFO,
173                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
174                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
175                         sum->s2length, (double)sum->flength);
176         }
177 }
178
179
180 /*
181  * Generate and send a stream of signatures/checksums that describe a buffer
182  *
183  * Generate approximately one checksum every block_len bytes.
184  */
185 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
186 {
187         size_t i;
188         struct map_struct *mapbuf;
189         struct sum_struct sum;
190         OFF_T offset = 0;
191
192         sum_sizes_sqroot(&sum, len);
193
194         if (len > 0)
195                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
196         else
197                 mapbuf = NULL;
198
199         write_sum_head(f_out, &sum);
200
201         for (i = 0; i < sum.count; i++) {
202                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
203                 char *map = map_ptr(mapbuf, offset, n1);
204                 uint32 sum1 = get_checksum1(map, n1);
205                 char sum2[SUM_LENGTH];
206
207                 if (f_copy >= 0)
208                         full_write(f_copy, map, n1);
209
210                 get_checksum2(map, n1, sum2);
211
212                 if (verbose > 3) {
213                         rprintf(FINFO,
214                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
215                                 (double)i, (double)offset, (long)n1,
216                                 (unsigned long)sum1);
217                 }
218                 write_int(f_out, sum1);
219                 write_buf(f_out, sum2, sum.s2length);
220                 len -= n1;
221                 offset += n1;
222         }
223
224         if (mapbuf)
225                 unmap_file(mapbuf);
226 }
227
228
229 /*
230  * Acts on file number @p i from @p flist, whose name is @p fname.
231  *
232  * First fixes up permissions, then generates checksums for the file.
233  *
234  * @note This comment was added later by mbp who was trying to work it
235  * out.  It might be wrong.
236  */
237 static void recv_generator(char *fname, struct file_list *flist,
238                            struct file_struct *file, int i,
239                            int f_out, int f_out_name)
240 {
241         int fd = -1, f_copy = -1;
242         STRUCT_STAT st, partial_st;
243         struct file_struct *back_file = NULL;
244         int statret, stat_errno;
245         char *fnamecmp, *partialptr, *backupptr = NULL;
246         char fnamecmpbuf[MAXPATHLEN];
247         uchar fnamecmp_type;
248
249         if (list_only)
250                 return;
251
252         if (verbose > 2)
253                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
254
255         if (server_exclude_list.head
256             && check_exclude(&server_exclude_list, fname,
257                              S_ISDIR(file->mode)) < 0) {
258                 if (verbose) {
259                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
260                                 safe_fname(fname));
261                 }
262                 return;
263         }
264
265         if (dry_run > 1) {
266                 statret = -1;
267                 stat_errno = ENOENT;
268         } else {
269                 statret = link_stat(fname, &st,
270                                     keep_dirlinks && S_ISDIR(file->mode));
271                 stat_errno = errno;
272         }
273
274         if (only_existing && statret == -1 && stat_errno == ENOENT) {
275                 /* we only want to update existing files */
276                 if (verbose > 1) {
277                         rprintf(FINFO, "not creating new file \"%s\"\n",
278                                 safe_fname(fname));
279                 }
280                 return;
281         }
282
283         if (statret == 0 && !preserve_perms
284             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
285                 /* if the file exists already and we aren't perserving
286                  * permissions then act as though the remote end sent
287                  * us the file permissions we already have */
288                 file->mode = (file->mode & ~CHMOD_BITS)
289                            | (st.st_mode & CHMOD_BITS);
290         }
291
292         if (S_ISDIR(file->mode)) {
293                 /* The file to be received is a directory, so we need
294                  * to prepare appropriately.  If there is already a
295                  * file of that name and it is *not* a directory, then
296                  * we need to delete it.  If it doesn't exist, then
297                  * (perhaps recursively) create it. */
298                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
299                         delete_file(fname, DEL_TERSE);
300                         statret = -1;
301                 }
302                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
303                         if (!relative_paths || errno != ENOENT
304                             || create_directory_path(fname, orig_umask) < 0
305                             || do_mkdir(fname, file->mode) < 0) {
306                                 rsyserr(FERROR, errno,
307                                         "recv_generator: mkdir %s failed",
308                                         full_fname(fname));
309                         }
310                 }
311                 /* f_out is set to -1 when doing final directory-permission
312                  * and modification-time repair. */
313                 if (set_perms(fname, file, statret ? NULL : &st, 0)
314                     && verbose && f_out != -1)
315                         rprintf(FINFO, "%s/\n", safe_fname(fname));
316                 if (delete_during && f_out != -1
317                     && (file->flags & FLAG_DEL_START))
318                         delete_in_dir(flist, fname);
319                 return;
320         } else if (max_size && file->length > max_size) {
321                 if (verbose > 1)
322                         rprintf(FINFO, "%s is over max-size\n", fname);
323                 return;
324         }
325
326         if (preserve_links && S_ISLNK(file->mode)) {
327 #if SUPPORT_LINKS
328                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
329                         if (verbose) {
330                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
331                                         full_fname(fname), file->u.link);
332                         }
333                         return;
334                 }
335                 if (statret == 0) {
336                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
337                         char lnk[MAXPATHLEN];
338                         int len;
339
340                         if (!dflag
341                             && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
342                                 lnk[len] = 0;
343                                 /* A link already pointing to the
344                                  * right place -- no further action
345                                  * required. */
346                                 if (strcmp(lnk, file->u.link) == 0) {
347                                         set_perms(fname, file, &st,
348                                                   PERMS_REPORT);
349                                         return;
350                                 }
351                         }
352                         /* Not the right symlink (or not a symlink), so
353                          * delete it. */
354                         delete_file(fname, dflag | DEL_TERSE);
355                 }
356                 if (do_symlink(file->u.link,fname) != 0) {
357                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
358                                 full_fname(fname), safe_fname(file->u.link));
359                 } else {
360                         set_perms(fname,file,NULL,0);
361                         if (verbose) {
362                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
363                                         safe_fname(file->u.link));
364                         }
365                 }
366 #endif
367                 return;
368         }
369
370         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
371                 if (statret != 0 ||
372                     st.st_mode != file->mode ||
373                     st.st_rdev != file->u.rdev) {
374                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
375                         delete_file(fname, dflag | DEL_TERSE);
376                         if (verbose > 2) {
377                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
378                                         safe_fname(fname),
379                                         (int)file->mode, (int)file->u.rdev);
380                         }
381                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
382                                 rsyserr(FERROR, errno, "mknod %s failed",
383                                         full_fname(fname));
384                         } else {
385                                 set_perms(fname,file,NULL,0);
386                                 if (verbose) {
387                                         rprintf(FINFO, "%s\n",
388                                                 safe_fname(fname));
389                                 }
390                         }
391                 } else {
392                         set_perms(fname, file, &st, PERMS_REPORT);
393                 }
394                 return;
395         }
396
397         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
398                 return;
399
400         if (!S_ISREG(file->mode)) {
401                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
402                         safe_fname(fname));
403                 return;
404         }
405
406         fnamecmp = fname;
407         fnamecmp_type = FNAMECMP_FNAME;
408
409         if (statret == -1 && basis_dir[0] != NULL) {
410                 int fallback_match = -1;
411                 int match_level = 0;
412                 int i = 0;
413                 do {
414                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
415                                  basis_dir[i], fname);
416                         if (link_stat(fnamecmpbuf, &st, 0) == 0
417                             && S_ISREG(st.st_mode)) {
418                                 statret = 0;
419                                 if (link_dest) {
420                                         if (!match_level) {
421                                                 fallback_match = i;
422                                                 match_level = 1;
423                                         } else if (match_level == 2
424                                             && !unchanged_attrs(file, &st))
425                                                 continue;
426                                         if (!unchanged_file(fnamecmpbuf, file, &st))
427                                                 continue;
428                                         fallback_match = i;
429                                         match_level = 2;
430                                         if (!unchanged_attrs(file, &st))
431                                                 continue;
432                                 }
433                                 match_level = 3;
434                                 break;
435                         }
436                 } while (basis_dir[++i] != NULL);
437                 if (statret == 0) {
438                         if (match_level < 3) {
439                                 i = fallback_match;
440                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
441                                          basis_dir[i], fname);
442                         }
443 #if HAVE_LINK
444                         if (link_dest && match_level == 3 && !dry_run) {
445                                 if (do_link(fnamecmpbuf, fname) < 0) {
446                                         if (verbose) {
447                                                 rsyserr(FINFO, errno,
448                                                         "link %s => %s",
449                                                         fnamecmpbuf,
450                                                         safe_fname(fname));
451                                         }
452                                         fnamecmp = fnamecmpbuf;
453                                         fnamecmp_type = i;
454                                 }
455                         } else
456 #endif
457                         {
458                                 fnamecmp = fnamecmpbuf;
459                                 fnamecmp_type = i;
460                         }
461                 }
462         }
463
464         if (statret == 0 && !S_ISREG(st.st_mode)) {
465                 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
466                 if (delete_file(fname, dflag | DEL_TERSE) != 0)
467                         return;
468                 statret = -1;
469                 stat_errno = ENOENT;
470         }
471
472         if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
473             && link_stat(partialptr, &partial_st, 0) == 0
474             && S_ISREG(partial_st.st_mode)) {
475                 if (statret == -1)
476                         goto prepare_to_open;
477         } else
478                 partialptr = NULL;
479
480         if (statret == -1) {
481                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
482                         return;
483                 if (stat_errno == ENOENT)
484                         goto notify_others;
485                 if (verbose > 1) {
486                         rsyserr(FERROR, stat_errno,
487                                 "recv_generator: failed to stat %s",
488                                 full_fname(fname));
489                 }
490                 return;
491         }
492
493         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
494                 if (verbose > 1)
495                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
496                 return;
497         }
498
499         if (update_only && fnamecmp_type == FNAMECMP_FNAME
500             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
501                 if (verbose > 1)
502                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
503                 return;
504         }
505
506         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
507                 ;
508         else if (unchanged_file(fnamecmp, file, &st)) {
509                 if (fnamecmp_type == FNAMECMP_FNAME)
510                         set_perms(fname, file, &st, PERMS_REPORT);
511                 return;
512         }
513
514 prepare_to_open:
515         if (partialptr) {
516                 st = partial_st;
517                 fnamecmp = partialptr;
518                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
519                 statret = 0;
520         }
521
522         if (dry_run || whole_file > 0) {
523                 statret = -1;
524                 goto notify_others;
525         }
526         if (read_batch)
527                 goto notify_others;
528
529         /* open the file */
530         fd = do_open(fnamecmp, O_RDONLY, 0);
531
532         if (fd == -1) {
533                 rsyserr(FERROR, errno, "failed to open %s, continuing",
534                         full_fname(fnamecmp));
535             pretend_missing:
536                 /* pretend the file didn't exist */
537                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
538                         return;
539                 statret = -1;
540                 goto notify_others;
541         }
542
543         if (inplace && make_backups) {
544                 if (!(backupptr = get_backup_name(fname))) {
545                         close(fd);
546                         return;
547                 }
548                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
549                         close(fd);
550                         goto pretend_missing;
551                 }
552                 if (robust_unlink(backupptr) && errno != ENOENT) {
553                         rsyserr(FERROR, errno, "unlink %s",
554                                 full_fname(backupptr));
555                         free(back_file);
556                         close(fd);
557                         return;
558                 }
559                 if ((f_copy = do_open(backupptr,
560                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
561                         rsyserr(FERROR, errno, "open %s",
562                                 full_fname(backupptr));
563                         free(back_file);
564                         close(fd);
565                         return;
566                 }
567                 fnamecmp_type = FNAMECMP_BACKUP;
568         }
569
570         if (verbose > 3) {
571                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
572                         safe_fname(fnamecmp), (double)st.st_size);
573         }
574
575         if (verbose > 2)
576                 rprintf(FINFO, "generating and sending sums for %d\n", i);
577
578 notify_others:
579         write_int(f_out, i);
580         if (protocol_version >= 29 && inplace && !read_batch)
581                 write_byte(f_out, fnamecmp_type);
582         if (f_out_name >= 0)
583                 write_byte(f_out_name, fnamecmp_type);
584
585         if (dry_run || read_batch)
586                 return;
587
588         if (statret == 0) {
589                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
590
591                 if (f_copy >= 0) {
592                         close(f_copy);
593                         set_perms(backupptr, back_file, NULL, 0);
594                         if (verbose > 1) {
595                                 rprintf(FINFO, "backed up %s to %s\n",
596                                         fname, backupptr);
597                         }
598                         free(back_file);
599                 }
600
601                 close(fd);
602         } else
603                 write_sum_head(f_out, NULL);
604 }
605
606
607 void generate_files(int f_out, struct file_list *flist, char *local_name,
608                     int f_out_name)
609 {
610         int i;
611         int phase = 0;
612         char fbuf[MAXPATHLEN];
613
614         if (verbose > 2) {
615                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
616                         (long)getpid(), flist->count);
617         }
618
619         if (verbose >= 2) {
620                 rprintf(FINFO,
621                         whole_file > 0
622                         ? "delta-transmission disabled for local transfer or --whole-file\n"
623                         : "delta transmission enabled\n");
624         }
625
626         /* we expect to just sit around now, so don't exit on a
627            timeout. If we really get a timeout then the other process should
628            exit */
629         io_timeout = 0;
630
631         for (i = 0; i < flist->count; i++) {
632                 struct file_struct *file = flist->files[i];
633                 struct file_struct copy;
634
635                 if (!file->basename)
636                         continue;
637                 /* we need to ensure that any directories we create have writeable
638                    permissions initially so that we can create the files within
639                    them. This is then fixed after the files are transferred */
640                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
641                         copy = *file;
642                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
643                          * handling of permissions is strange? */
644                         copy.mode |= S_IWUSR; /* user write */
645                         file = &copy;
646                 }
647
648                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
649                                flist, file, i, f_out, f_out_name);
650         }
651
652         phase++;
653         csum_length = SUM_LENGTH;
654         ignore_times = 1;
655
656         if (verbose > 2)
657                 rprintf(FINFO,"generate_files phase=%d\n",phase);
658
659         write_int(f_out, -1);
660
661         /* files can cycle through the system more than once
662          * to catch initial checksum errors */
663         while ((i = get_redo_num()) != -1) {
664                 struct file_struct *file = flist->files[i];
665                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
666                                flist, file, i, f_out, f_out_name);
667         }
668
669         phase++;
670         if (verbose > 2)
671                 rprintf(FINFO,"generate_files phase=%d\n",phase);
672
673         write_int(f_out, -1);
674
675         /* Read post-redo-phase MSG_DONE and any prior messages. */
676         get_redo_num();
677
678         if (preserve_hard_links)
679                 do_hard_links();
680
681         /* now we need to fix any directory permissions that were
682          * modified during the transfer */
683         for (i = 0; i < flist->count; i++) {
684                 struct file_struct *file = flist->files[i];
685                 if (!file->basename || !S_ISDIR(file->mode))
686                         continue;
687                 recv_generator(local_name ? local_name : f_name(file),
688                                flist, file, i, -1, -1);
689         }
690
691         if (verbose > 2)
692                 rprintf(FINFO,"generate_files finished\n");
693 }