- Moved write_sum_head() to io.c.
[rsync/rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int inplace;
40 extern int make_backups;
41 extern int csum_length;
42 extern int ignore_times;
43 extern int size_only;
44 extern OFF_T max_size;
45 extern int io_timeout;
46 extern int protocol_version;
47 extern int always_checksum;
48 extern char *partial_dir;
49 extern char *basis_dir[];
50 extern int compare_dest;
51 extern int link_dest;
52 extern int whole_file;
53 extern int local_server;
54 extern int list_only;
55 extern int read_batch;
56 extern int only_existing;
57 extern int orig_umask;
58 extern int safe_symlinks;
59 extern long block_size; /* "long" because popt can't set an int32. */
60
61 extern struct exclude_list_struct server_exclude_list;
62
63 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
64 {
65         if (preserve_perms
66          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
67                 return 0;
68
69         if (am_root && preserve_uid && st->st_uid != file->uid)
70                 return 0;
71
72         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
73                 return 0;
74
75         return 1;
76 }
77
78 /* Perform our quick-check heuristic for determining if a file is unchanged. */
79 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
80 {
81         if (st->st_size != file->length)
82                 return 0;
83
84         /* if always checksum is set then we use the checksum instead
85            of the file time to determine whether to sync */
86         if (always_checksum && S_ISREG(st->st_mode)) {
87                 char sum[MD4_SUM_LENGTH];
88                 file_checksum(fn, sum, st->st_size);
89                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
90                                                         : MD4_SUM_LENGTH) == 0;
91         }
92
93         if (size_only)
94                 return 1;
95
96         if (ignore_times)
97                 return 0;
98
99         return cmp_modtime(st->st_mtime, file->modtime) == 0;
100 }
101
102
103 /*
104  * set (initialize) the size entries in the per-file sum_struct
105  * calculating dynamic block and checksum sizes.
106  *
107  * This is only called from generate_and_send_sums() but is a separate
108  * function to encapsulate the logic.
109  *
110  * The block size is a rounded square root of file length.
111  *
112  * The checksum size is determined according to:
113  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
114  * provided by Donovan Baarda which gives a probability of rsync
115  * algorithm corrupting data and falling back using the whole md4
116  * checksums.
117  *
118  * This might be made one of several selectable heuristics.
119  */
120 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
121 {
122         int32 blength;
123         int s2length;
124
125         if (block_size)
126                 blength = block_size;
127         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
128                 blength = BLOCK_SIZE;
129         else {
130                 int32 c;
131                 uint64 l;
132                 int cnt;
133                 for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
134                 if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
135                         blength = MAX_BLOCK_SIZE;
136                 else {
137                     blength = 0;
138                     do {
139                             blength |= c;
140                             if (len < (uint64)blength * blength)
141                                     blength &= ~c;
142                             c >>= 1;
143                     } while (c >= 8);   /* round to multiple of 8 */
144                     blength = MAX(blength, BLOCK_SIZE);
145                 }
146         }
147
148         if (protocol_version < 27) {
149                 s2length = csum_length;
150         } else if (csum_length == SUM_LENGTH) {
151                 s2length = SUM_LENGTH;
152         } else {
153                 int32 c;
154                 uint64 l;
155                 int b = BLOCKSUM_BIAS;
156                 for (l = len; l >>= 1; b += 2) {}
157                 for (c = blength; c >>= 1 && b; b--) {}
158                 /* add a bit, subtract rollsum, round up. */
159                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
160                 s2length = MAX(s2length, csum_length);
161                 s2length = MIN(s2length, SUM_LENGTH);
162         }
163
164         sum->flength    = len;
165         sum->blength    = blength;
166         sum->s2length   = s2length;
167         sum->count      = (len + (blength - 1)) / blength;
168         sum->remainder  = (len % blength);
169
170         if (sum->count && verbose > 2) {
171                 rprintf(FINFO,
172                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
173                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
174                         sum->s2length, (double)sum->flength);
175         }
176 }
177
178
179 /*
180  * Generate and send a stream of signatures/checksums that describe a buffer
181  *
182  * Generate approximately one checksum every block_len bytes.
183  */
184 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
185 {
186         size_t i;
187         struct map_struct *mapbuf;
188         struct sum_struct sum;
189         OFF_T offset = 0;
190
191         sum_sizes_sqroot(&sum, len);
192
193         if (len > 0)
194                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
195         else
196                 mapbuf = NULL;
197
198         write_sum_head(f_out, &sum);
199
200         for (i = 0; i < sum.count; i++) {
201                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
202                 char *map = map_ptr(mapbuf, offset, n1);
203                 uint32 sum1 = get_checksum1(map, n1);
204                 char sum2[SUM_LENGTH];
205
206                 if (f_copy >= 0)
207                         full_write(f_copy, map, n1);
208
209                 get_checksum2(map, n1, sum2);
210
211                 if (verbose > 3) {
212                         rprintf(FINFO,
213                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
214                                 (double)i, (double)offset, (long)n1,
215                                 (unsigned long)sum1);
216                 }
217                 write_int(f_out, sum1);
218                 write_buf(f_out, sum2, sum.s2length);
219                 len -= n1;
220                 offset += n1;
221         }
222
223         if (mapbuf)
224                 unmap_file(mapbuf);
225 }
226
227
228 /*
229  * Acts on file number @p i from @p flist, whose name is @p fname.
230  *
231  * First fixes up permissions, then generates checksums for the file.
232  *
233  * @note This comment was added later by mbp who was trying to work it
234  * out.  It might be wrong.
235  */
236 static void recv_generator(char *fname, struct file_struct *file, int i,
237                            int f_out, int f_out_name)
238 {
239         int fd = -1, f_copy = -1;
240         STRUCT_STAT st, partial_st;
241         struct file_struct *back_file = NULL;
242         int statret, stat_errno;
243         char *fnamecmp, *partialptr, *backupptr = NULL;
244         char fnamecmpbuf[MAXPATHLEN];
245         uchar fnamecmp_type;
246
247         if (list_only)
248                 return;
249
250         if (verbose > 2)
251                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
252
253         if (server_exclude_list.head
254             && check_exclude(&server_exclude_list, fname,
255                              S_ISDIR(file->mode)) < 0) {
256                 if (verbose) {
257                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
258                                 safe_fname(fname));
259                 }
260                 return;
261         }
262
263         if (dry_run > 1) {
264                 statret = -1;
265                 stat_errno = ENOENT;
266         } else {
267                 statret = link_stat(fname, &st,
268                                     keep_dirlinks && S_ISDIR(file->mode));
269                 stat_errno = errno;
270         }
271
272         if (only_existing && statret == -1 && stat_errno == ENOENT) {
273                 /* we only want to update existing files */
274                 if (verbose > 1) {
275                         rprintf(FINFO, "not creating new file \"%s\"\n",
276                                 safe_fname(fname));
277                 }
278                 return;
279         }
280
281         if (statret == 0 && !preserve_perms
282             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
283                 /* if the file exists already and we aren't perserving
284                  * permissions then act as though the remote end sent
285                  * us the file permissions we already have */
286                 file->mode = (file->mode & ~CHMOD_BITS)
287                            | (st.st_mode & CHMOD_BITS);
288         }
289
290         if (S_ISDIR(file->mode)) {
291                 /* The file to be received is a directory, so we need
292                  * to prepare appropriately.  If there is already a
293                  * file of that name and it is *not* a directory, then
294                  * we need to delete it.  If it doesn't exist, then
295                  * recursively create it. */
296
297                 if (dry_run)
298                         return; /* TODO: causes inaccuracies -- fix */
299                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
300                         delete_file(fname, DEL_TERSE);
301                         statret = -1;
302                 }
303                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
304                         if (!(relative_paths && errno == ENOENT
305                             && create_directory_path(fname, orig_umask) == 0
306                             && do_mkdir(fname, file->mode) == 0)) {
307                                 rsyserr(FERROR, errno,
308                                         "recv_generator: mkdir %s failed",
309                                         full_fname(fname));
310                         }
311                 }
312                 /* f_out is set to -1 when doing final directory-permission
313                  * and modification-time repair. */
314                 if (set_perms(fname, file, statret ? NULL : &st, 0)
315                     && verbose && f_out != -1)
316                         rprintf(FINFO, "%s/\n", safe_fname(fname));
317                 return;
318         } else if (max_size && file->length > max_size) {
319                 if (verbose > 1)
320                         rprintf(FINFO, "%s is over max-size\n", fname);
321                 return;
322         }
323
324         if (preserve_links && S_ISLNK(file->mode)) {
325 #if SUPPORT_LINKS
326                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
327                         if (verbose) {
328                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
329                                         full_fname(fname), file->u.link);
330                         }
331                         return;
332                 }
333                 if (statret == 0) {
334                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
335                         char lnk[MAXPATHLEN];
336                         int len;
337
338                         if (!dflag
339                             && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
340                                 lnk[len] = 0;
341                                 /* A link already pointing to the
342                                  * right place -- no further action
343                                  * required. */
344                                 if (strcmp(lnk, file->u.link) == 0) {
345                                         set_perms(fname, file, &st,
346                                                   PERMS_REPORT);
347                                         return;
348                                 }
349                         }
350                         /* Not the right symlink (or not a symlink), so
351                          * delete it. */
352                         delete_file(fname, dflag | DEL_TERSE);
353                 }
354                 if (do_symlink(file->u.link,fname) != 0) {
355                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
356                                 full_fname(fname), safe_fname(file->u.link));
357                 } else {
358                         set_perms(fname,file,NULL,0);
359                         if (verbose) {
360                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
361                                         safe_fname(file->u.link));
362                         }
363                 }
364 #endif
365                 return;
366         }
367
368         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
369                 if (statret != 0 ||
370                     st.st_mode != file->mode ||
371                     st.st_rdev != file->u.rdev) {
372                         int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
373                         delete_file(fname, dflag | DEL_TERSE);
374                         if (verbose > 2) {
375                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
376                                         safe_fname(fname),
377                                         (int)file->mode, (int)file->u.rdev);
378                         }
379                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
380                                 rsyserr(FERROR, errno, "mknod %s failed",
381                                         full_fname(fname));
382                         } else {
383                                 set_perms(fname,file,NULL,0);
384                                 if (verbose) {
385                                         rprintf(FINFO, "%s\n",
386                                                 safe_fname(fname));
387                                 }
388                         }
389                 } else {
390                         set_perms(fname, file, &st, PERMS_REPORT);
391                 }
392                 return;
393         }
394
395         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
396                 return;
397
398         if (!S_ISREG(file->mode)) {
399                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
400                         safe_fname(fname));
401                 return;
402         }
403
404         fnamecmp = fname;
405         fnamecmp_type = FNAMECMP_FNAME;
406
407         if (statret == -1 && basis_dir[0] != NULL) {
408                 int fallback_match = -1;
409                 int match_level = 0;
410                 int i = 0;
411                 do {
412                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
413                                  basis_dir[i], fname);
414                         if (link_stat(fnamecmpbuf, &st, 0) == 0
415                             && S_ISREG(st.st_mode)) {
416                                 statret = 0;
417                                 if (link_dest) {
418                                         if (!match_level) {
419                                                 fallback_match = i;
420                                                 match_level = 1;
421                                         } else if (match_level == 2
422                                             && !unchanged_attrs(file, &st))
423                                                 continue;
424                                         if (!unchanged_file(fnamecmpbuf, file, &st))
425                                                 continue;
426                                         fallback_match = i;
427                                         match_level = 2;
428                                         if (!unchanged_attrs(file, &st))
429                                                 continue;
430                                 }
431                                 match_level = 3;
432                                 break;
433                         }
434                 } while (basis_dir[++i] != NULL);
435                 if (statret == 0) {
436                         if (match_level < 3) {
437                                 i = fallback_match;
438                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
439                                          basis_dir[i], fname);
440                         }
441 #if HAVE_LINK
442                         if (link_dest && match_level == 3 && !dry_run) {
443                                 if (do_link(fnamecmpbuf, fname) < 0) {
444                                         if (verbose) {
445                                                 rsyserr(FINFO, errno,
446                                                         "link %s => %s",
447                                                         fnamecmpbuf,
448                                                         safe_fname(fname));
449                                         }
450                                         fnamecmp = fnamecmpbuf;
451                                         fnamecmp_type = i;
452                                 }
453                         } else
454 #endif
455                         {
456                                 fnamecmp = fnamecmpbuf;
457                                 fnamecmp_type = i;
458                         }
459                 }
460         }
461
462         if (statret == 0 && !S_ISREG(st.st_mode)) {
463                 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
464                 if (delete_file(fname, dflag | DEL_TERSE) != 0)
465                         return;
466                 statret = -1;
467                 stat_errno = ENOENT;
468         }
469
470         if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
471             && link_stat(partialptr, &partial_st, 0) == 0
472             && S_ISREG(partial_st.st_mode)) {
473                 if (statret == -1)
474                         goto prepare_to_open;
475         } else
476                 partialptr = NULL;
477
478         if (statret == -1) {
479                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
480                         return;
481                 if (stat_errno == ENOENT)
482                         goto notify_others;
483                 if (verbose > 1) {
484                         rsyserr(FERROR, stat_errno,
485                                 "recv_generator: failed to stat %s",
486                                 full_fname(fname));
487                 }
488                 return;
489         }
490
491         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
492                 if (verbose > 1)
493                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
494                 return;
495         }
496
497         if (update_only && fnamecmp_type == FNAMECMP_FNAME
498             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
499                 if (verbose > 1)
500                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
501                 return;
502         }
503
504         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
505                 ;
506         else if (unchanged_file(fnamecmp, file, &st)) {
507                 if (fnamecmp_type == FNAMECMP_FNAME)
508                         set_perms(fname, file, &st, PERMS_REPORT);
509                 return;
510         }
511
512 prepare_to_open:
513         if (partialptr) {
514                 st = partial_st;
515                 fnamecmp = partialptr;
516                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
517                 statret = 0;
518         }
519
520         if (dry_run || whole_file > 0) {
521                 statret = -1;
522                 goto notify_others;
523         }
524         if (read_batch)
525                 goto notify_others;
526
527         /* open the file */
528         fd = do_open(fnamecmp, O_RDONLY, 0);
529
530         if (fd == -1) {
531                 rsyserr(FERROR, errno, "failed to open %s, continuing",
532                         full_fname(fnamecmp));
533             pretend_missing:
534                 /* pretend the file didn't exist */
535                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
536                         return;
537                 statret = -1;
538                 goto notify_others;
539         }
540
541         if (inplace && make_backups) {
542                 if (!(backupptr = get_backup_name(fname))) {
543                         close(fd);
544                         return;
545                 }
546                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
547                         close(fd);
548                         goto pretend_missing;
549                 }
550                 if (robust_unlink(backupptr) && errno != ENOENT) {
551                         rsyserr(FERROR, errno, "unlink %s",
552                                 full_fname(backupptr));
553                         free(back_file);
554                         close(fd);
555                         return;
556                 }
557                 if ((f_copy = do_open(backupptr,
558                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
559                         rsyserr(FERROR, errno, "open %s",
560                                 full_fname(backupptr));
561                         free(back_file);
562                         close(fd);
563                         return;
564                 }
565                 fnamecmp_type = FNAMECMP_BACKUP;
566         }
567
568         if (verbose > 3) {
569                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
570                         safe_fname(fnamecmp), (double)st.st_size);
571         }
572
573         if (verbose > 2)
574                 rprintf(FINFO, "generating and sending sums for %d\n", i);
575
576 notify_others:
577         write_int(f_out, i);
578         if (protocol_version >= 29 && inplace && !read_batch)
579                 write_byte(f_out, fnamecmp_type);
580         if (f_out_name >= 0)
581                 write_byte(f_out_name, fnamecmp_type);
582
583         if (dry_run || read_batch)
584                 return;
585
586         if (statret == 0) {
587                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
588
589                 if (f_copy >= 0) {
590                         close(f_copy);
591                         set_perms(backupptr, back_file, NULL, 0);
592                         if (verbose > 1) {
593                                 rprintf(FINFO, "backed up %s to %s\n",
594                                         fname, backupptr);
595                         }
596                         free(back_file);
597                 }
598
599                 close(fd);
600         } else
601                 write_sum_head(f_out, NULL);
602 }
603
604
605 void generate_files(int f_out, struct file_list *flist, char *local_name,
606                     int f_out_name)
607 {
608         int i;
609         int phase = 0;
610         char fbuf[MAXPATHLEN];
611
612         if (verbose > 2) {
613                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
614                         (long)getpid(), flist->count);
615         }
616
617         if (verbose >= 2) {
618                 rprintf(FINFO,
619                         whole_file > 0
620                         ? "delta-transmission disabled for local transfer or --whole-file\n"
621                         : "delta transmission enabled\n");
622         }
623
624         /* we expect to just sit around now, so don't exit on a
625            timeout. If we really get a timeout then the other process should
626            exit */
627         io_timeout = 0;
628
629         for (i = 0; i < flist->count; i++) {
630                 struct file_struct *file = flist->files[i];
631                 struct file_struct copy;
632
633                 if (!file->basename)
634                         continue;
635                 /* we need to ensure that any directories we create have writeable
636                    permissions initially so that we can create the files within
637                    them. This is then fixed after the files are transferred */
638                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
639                         copy = *file;
640                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
641                          * handling of permissions is strange? */
642                         copy.mode |= S_IWUSR; /* user write */
643                         file = &copy;
644                 }
645
646                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
647                                file, i, f_out, f_out_name);
648         }
649
650         phase++;
651         csum_length = SUM_LENGTH;
652         ignore_times = 1;
653
654         if (verbose > 2)
655                 rprintf(FINFO,"generate_files phase=%d\n",phase);
656
657         write_int(f_out, -1);
658
659         /* files can cycle through the system more than once
660          * to catch initial checksum errors */
661         while ((i = get_redo_num()) != -1) {
662                 struct file_struct *file = flist->files[i];
663                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
664                                file, i, f_out, f_out_name);
665         }
666
667         phase++;
668         if (verbose > 2)
669                 rprintf(FINFO,"generate_files phase=%d\n",phase);
670
671         write_int(f_out, -1);
672
673         /* Read post-redo-phase MSG_DONE and any prior messages. */
674         get_redo_num();
675
676         if (preserve_hard_links)
677                 do_hard_links();
678
679         /* now we need to fix any directory permissions that were
680          * modified during the transfer */
681         for (i = 0; i < flist->count; i++) {
682                 struct file_struct *file = flist->files[i];
683                 if (!file->basename || !S_ISDIR(file->mode))
684                         continue;
685                 recv_generator(local_name ? local_name : f_name(file),
686                                file, i, -1, -1);
687         }
688
689         if (verbose > 2)
690                 rprintf(FINFO,"generate_files finished\n");
691 }