- Changed the map_size parameter to map_file() to be an int32 named
[rsync/rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int inplace;
40 extern int make_backups;
41 extern int csum_length;
42 extern int ignore_times;
43 extern int size_only;
44 extern OFF_T max_size;
45 extern int io_timeout;
46 extern int protocol_version;
47 extern int always_checksum;
48 extern char *partial_dir;
49 extern char *basis_dir[];
50 extern int compare_dest;
51 extern int link_dest;
52 extern int whole_file;
53 extern int local_server;
54 extern int list_only;
55 extern int read_batch;
56 extern int only_existing;
57 extern int orig_umask;
58 extern int safe_symlinks;
59 extern long block_size; /* "long" because popt can't set an int32. */
60
61 extern struct exclude_list_struct server_exclude_list;
62
63 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
64 {
65         if (preserve_perms
66          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
67                 return 0;
68
69         if (am_root && preserve_uid && st->st_uid != file->uid)
70                 return 0;
71
72         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
73                 return 0;
74
75         return 1;
76 }
77
78 /* Perform our quick-check heuristic for determining if a file is unchanged. */
79 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
80 {
81         if (st->st_size != file->length)
82                 return 0;
83
84         /* if always checksum is set then we use the checksum instead
85            of the file time to determine whether to sync */
86         if (always_checksum && S_ISREG(st->st_mode)) {
87                 char sum[MD4_SUM_LENGTH];
88                 file_checksum(fn, sum, st->st_size);
89                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
90                                                         : MD4_SUM_LENGTH) == 0;
91         }
92
93         if (size_only)
94                 return 1;
95
96         if (ignore_times)
97                 return 0;
98
99         return cmp_modtime(st->st_mtime, file->modtime) == 0;
100 }
101
102
103 /*
104  * NULL sum_struct means we have no checksums
105  */
106 void write_sum_head(int f, struct sum_struct *sum)
107 {
108         static struct sum_struct null_sum;
109
110         if (sum == NULL)
111                 sum = &null_sum;
112
113         write_int(f, sum->count);
114         write_int(f, sum->blength);
115         if (protocol_version >= 27)
116                 write_int(f, sum->s2length);
117         write_int(f, sum->remainder);
118 }
119
120 /*
121  * set (initialize) the size entries in the per-file sum_struct
122  * calculating dynamic block and checksum sizes.
123  *
124  * This is only called from generate_and_send_sums() but is a separate
125  * function to encapsulate the logic.
126  *
127  * The block size is a rounded square root of file length.
128  *
129  * The checksum size is determined according to:
130  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
131  * provided by Donovan Baarda which gives a probability of rsync
132  * algorithm corrupting data and falling back using the whole md4
133  * checksums.
134  *
135  * This might be made one of several selectable heuristics.
136  */
137 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
138 {
139         int32 blength;
140         int s2length;
141
142         if (block_size)
143                 blength = block_size;
144         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
145                 blength = BLOCK_SIZE;
146         else {
147                 int32 c;
148                 uint64 l;
149                 for (c = 1, l = len; l >>= 2; c <<= 1) {
150                     assert(c > 0);
151                 }
152                 blength = 0;
153                 do {
154                         blength |= c;
155                         if (len < (uint64)blength * blength)
156                                 blength &= ~c;
157                         c >>= 1;
158                 } while (c >= 8);       /* round to multiple of 8 */
159                 blength = MAX(blength, BLOCK_SIZE);
160         }
161
162         if (protocol_version < 27) {
163                 s2length = csum_length;
164         } else if (csum_length == SUM_LENGTH) {
165                 s2length = SUM_LENGTH;
166         } else {
167                 int32 c;
168                 uint64 l;
169                 int b = BLOCKSUM_BIAS;
170                 for (l = len; l >>= 1; b += 2) {}
171                 for (c = blength; c >>= 1 && b; b--) {}
172                 /* add a bit, subtract rollsum, round up. */
173                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
174                 s2length = MAX(s2length, csum_length);
175                 s2length = MIN(s2length, SUM_LENGTH);
176         }
177
178         sum->flength    = len;
179         sum->blength    = blength;
180         sum->s2length   = s2length;
181         sum->count      = (len + (blength - 1)) / blength;
182         sum->remainder  = (len % blength);
183
184         if (sum->count && verbose > 2) {
185                 rprintf(FINFO,
186                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
187                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
188                         sum->s2length, (double)sum->flength);
189         }
190 }
191
192
193 /*
194  * Generate and send a stream of signatures/checksums that describe a buffer
195  *
196  * Generate approximately one checksum every block_len bytes.
197  */
198 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
199 {
200         size_t i;
201         struct map_struct *mapbuf;
202         struct sum_struct sum;
203         OFF_T offset = 0;
204
205         sum_sizes_sqroot(&sum, len);
206
207         if (len > 0)
208                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
209         else
210                 mapbuf = NULL;
211
212         write_sum_head(f_out, &sum);
213
214         for (i = 0; i < sum.count; i++) {
215                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
216                 char *map = map_ptr(mapbuf, offset, n1);
217                 uint32 sum1 = get_checksum1(map, n1);
218                 char sum2[SUM_LENGTH];
219
220                 if (f_copy >= 0)
221                         full_write(f_copy, map, n1);
222
223                 get_checksum2(map, n1, sum2);
224
225                 if (verbose > 3) {
226                         rprintf(FINFO,
227                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
228                                 (double)i, (double)offset, (long)n1,
229                                 (unsigned long)sum1);
230                 }
231                 write_int(f_out, sum1);
232                 write_buf(f_out, sum2, sum.s2length);
233                 len -= n1;
234                 offset += n1;
235         }
236
237         if (mapbuf)
238                 unmap_file(mapbuf);
239 }
240
241
242
243 /*
244  * Acts on file number @p i from @p flist, whose name is @p fname.
245  *
246  * First fixes up permissions, then generates checksums for the file.
247  *
248  * @note This comment was added later by mbp who was trying to work it
249  * out.  It might be wrong.
250  */
251 static void recv_generator(char *fname, struct file_struct *file, int i,
252                            int f_out, int f_out_name)
253 {
254         int fd = -1, f_copy = -1;
255         STRUCT_STAT st, partial_st;
256         struct file_struct *back_file = NULL;
257         int statret, stat_errno;
258         char *fnamecmp, *partialptr, *backupptr = NULL;
259         char fnamecmpbuf[MAXPATHLEN];
260         uchar fnamecmp_type;
261
262         if (list_only)
263                 return;
264
265         if (verbose > 2)
266                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
267
268         if (server_exclude_list.head
269             && check_exclude(&server_exclude_list, fname,
270                              S_ISDIR(file->mode)) < 0) {
271                 if (verbose) {
272                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
273                                 safe_fname(fname));
274                 }
275                 return;
276         }
277
278         if (dry_run > 1) {
279                 statret = -1;
280                 stat_errno = ENOENT;
281         } else {
282                 statret = link_stat(fname, &st,
283                                     keep_dirlinks && S_ISDIR(file->mode));
284                 stat_errno = errno;
285         }
286
287         if (only_existing && statret == -1 && stat_errno == ENOENT) {
288                 /* we only want to update existing files */
289                 if (verbose > 1) {
290                         rprintf(FINFO, "not creating new file \"%s\"\n",
291                                 safe_fname(fname));
292                 }
293                 return;
294         }
295
296         if (statret == 0 && !preserve_perms
297             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
298                 /* if the file exists already and we aren't perserving
299                  * permissions then act as though the remote end sent
300                  * us the file permissions we already have */
301                 file->mode = (file->mode & ~CHMOD_BITS)
302                            | (st.st_mode & CHMOD_BITS);
303         }
304
305         if (S_ISDIR(file->mode)) {
306                 /* The file to be received is a directory, so we need
307                  * to prepare appropriately.  If there is already a
308                  * file of that name and it is *not* a directory, then
309                  * we need to delete it.  If it doesn't exist, then
310                  * recursively create it. */
311
312                 if (dry_run)
313                         return; /* TODO: causes inaccuracies -- fix */
314                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
315                         if (robust_unlink(fname) != 0) {
316                                 rsyserr(FERROR, errno,
317                                         "recv_generator: unlink %s to make room for directory",
318                                         full_fname(fname));
319                                 return;
320                         }
321                         statret = -1;
322                 }
323                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
324                         if (!(relative_paths && errno == ENOENT
325                             && create_directory_path(fname, orig_umask) == 0
326                             && do_mkdir(fname, file->mode) == 0)) {
327                                 rsyserr(FERROR, errno,
328                                         "recv_generator: mkdir %s failed",
329                                         full_fname(fname));
330                         }
331                 }
332                 /* f_out is set to -1 when doing final directory-permission
333                  * and modification-time repair. */
334                 if (set_perms(fname, file, statret ? NULL : &st, 0)
335                     && verbose && f_out != -1)
336                         rprintf(FINFO, "%s/\n", safe_fname(fname));
337                 return;
338         } else if (max_size && file->length > max_size) {
339                 if (verbose > 1)
340                         rprintf(FINFO, "%s is over max-size\n", fname);
341                 return;
342         }
343
344         if (preserve_links && S_ISLNK(file->mode)) {
345 #if SUPPORT_LINKS
346                 char lnk[MAXPATHLEN];
347                 int l;
348
349                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
350                         if (verbose) {
351                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
352                                         full_fname(fname), file->u.link);
353                         }
354                         return;
355                 }
356                 if (statret == 0) {
357                         l = readlink(fname,lnk,MAXPATHLEN-1);
358                         if (l > 0) {
359                                 lnk[l] = 0;
360                                 /* A link already pointing to the
361                                  * right place -- no further action
362                                  * required. */
363                                 if (strcmp(lnk,file->u.link) == 0) {
364                                         set_perms(fname, file, &st,
365                                                   PERMS_REPORT);
366                                         return;
367                                 }
368                         }
369                         /* Not a symlink, so delete whatever's
370                          * already there and put a new symlink
371                          * in place. */
372                         delete_file(fname);
373                 }
374                 if (do_symlink(file->u.link,fname) != 0) {
375                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
376                                 full_fname(fname), safe_fname(file->u.link));
377                 } else {
378                         set_perms(fname,file,NULL,0);
379                         if (verbose) {
380                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
381                                         safe_fname(file->u.link));
382                         }
383                 }
384 #endif
385                 return;
386         }
387
388         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
389                 if (statret != 0 ||
390                     st.st_mode != file->mode ||
391                     st.st_rdev != file->u.rdev) {
392                         delete_file(fname);
393                         if (verbose > 2) {
394                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
395                                         safe_fname(fname),
396                                         (int)file->mode, (int)file->u.rdev);
397                         }
398                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
399                                 rsyserr(FERROR, errno, "mknod %s failed",
400                                         full_fname(fname));
401                         } else {
402                                 set_perms(fname,file,NULL,0);
403                                 if (verbose) {
404                                         rprintf(FINFO, "%s\n",
405                                                 safe_fname(fname));
406                                 }
407                         }
408                 } else {
409                         set_perms(fname, file, &st, PERMS_REPORT);
410                 }
411                 return;
412         }
413
414         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
415                 return;
416
417         if (!S_ISREG(file->mode)) {
418                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
419                         safe_fname(fname));
420                 return;
421         }
422
423         fnamecmp = fname;
424         fnamecmp_type = FNAMECMP_FNAME;
425
426         if (statret == -1 && basis_dir[0] != NULL) {
427                 int fallback_match = -1;
428                 int match_level = 0;
429                 int i = 0;
430                 do {
431                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
432                                  basis_dir[i], fname);
433                         if (link_stat(fnamecmpbuf, &st, 0) == 0
434                             && S_ISREG(st.st_mode)) {
435                                 statret = 0;
436                                 if (link_dest) {
437                                         if (!match_level) {
438                                                 fallback_match = i;
439                                                 match_level = 1;
440                                         } else if (match_level == 2
441                                             && !unchanged_attrs(file, &st))
442                                                 continue;
443                                         if (!unchanged_file(fnamecmpbuf, file, &st))
444                                                 continue;
445                                         fallback_match = i;
446                                         match_level = 2;
447                                         if (!unchanged_attrs(file, &st))
448                                                 continue;
449                                 }
450                                 match_level = 3;
451                                 break;
452                         }
453                 } while (basis_dir[++i] != NULL);
454                 if (statret == 0) {
455                         if (match_level < 3) {
456                                 i = fallback_match;
457                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
458                                          basis_dir[i], fname);
459                         }
460 #if HAVE_LINK
461                         if (link_dest && match_level == 3 && !dry_run) {
462                                 if (do_link(fnamecmpbuf, fname) < 0) {
463                                         if (verbose) {
464                                                 rsyserr(FINFO, errno,
465                                                         "link %s => %s",
466                                                         fnamecmpbuf,
467                                                         safe_fname(fname));
468                                         }
469                                         fnamecmp = fnamecmpbuf;
470                                         fnamecmp_type = i;
471                                 }
472                         } else
473 #endif
474                         {
475                                 fnamecmp = fnamecmpbuf;
476                                 fnamecmp_type = i;
477                         }
478                 }
479         }
480
481         if (statret == 0 && !S_ISREG(st.st_mode)) {
482                 if (delete_file(fname) != 0)
483                         return;
484                 statret = -1;
485                 stat_errno = ENOENT;
486         }
487
488         if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
489             && link_stat(partialptr, &partial_st, 0) == 0
490             && S_ISREG(partial_st.st_mode)) {
491                 if (statret == -1)
492                         goto prepare_to_open;
493         } else
494                 partialptr = NULL;
495
496         if (statret == -1) {
497                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
498                         return;
499                 if (stat_errno == ENOENT)
500                         goto notify_others;
501                 if (verbose > 1) {
502                         rsyserr(FERROR, stat_errno,
503                                 "recv_generator: failed to stat %s",
504                                 full_fname(fname));
505                 }
506                 return;
507         }
508
509         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
510                 if (verbose > 1)
511                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
512                 return;
513         }
514
515         if (update_only && fnamecmp_type == FNAMECMP_FNAME
516             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
517                 if (verbose > 1)
518                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
519                 return;
520         }
521
522         if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
523                 ;
524         else if (unchanged_file(fnamecmp, file, &st)) {
525                 if (fnamecmp_type == FNAMECMP_FNAME)
526                         set_perms(fname, file, &st, PERMS_REPORT);
527                 return;
528         }
529
530 prepare_to_open:
531         if (partialptr) {
532                 st = partial_st;
533                 fnamecmp = partialptr;
534                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
535                 statret = 0;
536         }
537
538         if (dry_run || whole_file > 0) {
539                 statret = -1;
540                 goto notify_others;
541         }
542         if (read_batch)
543                 goto notify_others;
544
545         /* open the file */
546         fd = do_open(fnamecmp, O_RDONLY, 0);
547
548         if (fd == -1) {
549                 rsyserr(FERROR, errno, "failed to open %s, continuing",
550                         full_fname(fnamecmp));
551             pretend_missing:
552                 /* pretend the file didn't exist */
553                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
554                         return;
555                 statret = -1;
556                 goto notify_others;
557         }
558
559         if (inplace && make_backups) {
560                 if (!(backupptr = get_backup_name(fname))) {
561                         close(fd);
562                         return;
563                 }
564                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
565                         close(fd);
566                         goto pretend_missing;
567                 }
568                 if (robust_unlink(backupptr) && errno != ENOENT) {
569                         rsyserr(FERROR, errno, "unlink %s",
570                                 full_fname(backupptr));
571                         free(back_file);
572                         close(fd);
573                         return;
574                 }
575                 if ((f_copy = do_open(backupptr,
576                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
577                         rsyserr(FERROR, errno, "open %s",
578                                 full_fname(backupptr));
579                         free(back_file);
580                         close(fd);
581                         return;
582                 }
583                 fnamecmp_type = FNAMECMP_BACKUP;
584         }
585
586         if (verbose > 3) {
587                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
588                         safe_fname(fnamecmp), (double)st.st_size);
589         }
590
591         if (verbose > 2)
592                 rprintf(FINFO, "generating and sending sums for %d\n", i);
593
594 notify_others:
595         write_int(f_out, i);
596         if (f_out_name >= 0)
597                 write_byte(f_out_name, fnamecmp_type);
598
599         if (dry_run || read_batch)
600                 return;
601
602         if (statret == 0) {
603                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
604
605                 if (f_copy >= 0) {
606                         close(f_copy);
607                         set_perms(backupptr, back_file, NULL, 0);
608                         if (verbose > 1) {
609                                 rprintf(FINFO, "backed up %s to %s\n",
610                                         fname, backupptr);
611                         }
612                         free(back_file);
613                 }
614
615                 close(fd);
616         } else
617                 write_sum_head(f_out, NULL);
618 }
619
620
621 void generate_files(int f_out, struct file_list *flist, char *local_name,
622                     int f_out_name)
623 {
624         int i;
625         int phase = 0;
626         char fbuf[MAXPATHLEN];
627
628         if (verbose > 2) {
629                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
630                         (long)getpid(), flist->count);
631         }
632
633         if (verbose >= 2) {
634                 rprintf(FINFO,
635                         whole_file > 0
636                         ? "delta-transmission disabled for local transfer or --whole-file\n"
637                         : "delta transmission enabled\n");
638         }
639
640         /* we expect to just sit around now, so don't exit on a
641            timeout. If we really get a timeout then the other process should
642            exit */
643         io_timeout = 0;
644
645         for (i = 0; i < flist->count; i++) {
646                 struct file_struct *file = flist->files[i];
647                 struct file_struct copy;
648
649                 if (!file->basename)
650                         continue;
651                 /* we need to ensure that any directories we create have writeable
652                    permissions initially so that we can create the files within
653                    them. This is then fixed after the files are transferred */
654                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
655                         copy = *file;
656                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
657                          * handling of permissions is strange? */
658                         copy.mode |= S_IWUSR; /* user write */
659                         file = &copy;
660                 }
661
662                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
663                                file, i, f_out, f_out_name);
664         }
665
666         phase++;
667         csum_length = SUM_LENGTH;
668         ignore_times = 1;
669
670         if (verbose > 2)
671                 rprintf(FINFO,"generate_files phase=%d\n",phase);
672
673         write_int(f_out, -1);
674
675         /* files can cycle through the system more than once
676          * to catch initial checksum errors */
677         while ((i = get_redo_num()) != -1) {
678                 struct file_struct *file = flist->files[i];
679                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
680                                file, i, f_out, f_out_name);
681         }
682
683         phase++;
684         if (verbose > 2)
685                 rprintf(FINFO,"generate_files phase=%d\n",phase);
686
687         write_int(f_out, -1);
688
689         /* Read post-redo-phase MSG_DONE and any prior messages. */
690         get_redo_num();
691
692         if (preserve_hard_links)
693                 do_hard_links();
694
695         /* now we need to fix any directory permissions that were
696          * modified during the transfer */
697         for (i = 0; i < flist->count; i++) {
698                 struct file_struct *file = flist->files[i];
699                 if (!file->basename || !S_ISDIR(file->mode))
700                         continue;
701                 recv_generator(local_name ? local_name : f_name(file),
702                                file, i, -1, -1);
703         }
704
705         if (verbose > 2)
706                 rprintf(FINFO,"generate_files finished\n");
707 }