We also need the "name pipe" from the generator to the receiver
[rsync/rsync.git] / generator.c
1 /* -*- c-file-style: "linux" -*-
2
3    rsync -- fast file replication program
4
5    Copyright (C) 1996-2000 by Andrew Tridgell
6    Copyright (C) Paul Mackerras 1996
7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9    This program is free software; you can redistribute it and/or modify
10    it under the terms of the GNU General Public License as published by
11    the Free Software Foundation; either version 2 of the License, or
12    (at your option) any later version.
13
14    This program is distributed in the hope that it will be useful,
15    but WITHOUT ANY WARRANTY; without even the implied warranty of
16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17    GNU General Public License for more details.
18
19    You should have received a copy of the GNU General Public License
20    along with this program; if not, write to the Free Software
21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 */
23
24 #include "rsync.h"
25
26 extern int verbose;
27 extern int dry_run;
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
31 extern int am_root;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int inplace;
40 extern int make_backups;
41 extern int csum_length;
42 extern int ignore_times;
43 extern int size_only;
44 extern OFF_T max_size;
45 extern int io_timeout;
46 extern int protocol_version;
47 extern int always_checksum;
48 extern char *partial_dir;
49 extern char *basis_dir[];
50 extern int copy_dest;
51 extern int link_dest;
52 extern int whole_file;
53 extern int local_server;
54 extern int list_only;
55 extern int read_batch;
56 extern int only_existing;
57 extern int orig_umask;
58 extern int safe_symlinks;
59 extern long block_size; /* "long" because popt can't set an int32. */
60
61 extern struct exclude_list_struct server_exclude_list;
62
63 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
64 {
65         if (preserve_perms
66          && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
67                 return 0;
68
69         if (am_root && preserve_uid && st->st_uid != file->uid)
70                 return 0;
71
72         if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
73                 return 0;
74
75         return 1;
76 }
77
78 /* Perform our quick-check heuristic for determining if a file is unchanged. */
79 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
80 {
81         if (st->st_size != file->length)
82                 return 0;
83
84         /* if always checksum is set then we use the checksum instead
85            of the file time to determine whether to sync */
86         if (always_checksum && S_ISREG(st->st_mode)) {
87                 char sum[MD4_SUM_LENGTH];
88                 file_checksum(fn, sum, st->st_size);
89                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
90                                                         : MD4_SUM_LENGTH) == 0;
91         }
92
93         if (size_only)
94                 return 1;
95
96         if (ignore_times)
97                 return 0;
98
99         return cmp_modtime(st->st_mtime, file->modtime) == 0;
100 }
101
102
103 /*
104  * NULL sum_struct means we have no checksums
105  */
106 void write_sum_head(int f, struct sum_struct *sum)
107 {
108         static struct sum_struct null_sum;
109
110         if (sum == NULL)
111                 sum = &null_sum;
112
113         write_int(f, sum->count);
114         write_int(f, sum->blength);
115         if (protocol_version >= 27)
116                 write_int(f, sum->s2length);
117         write_int(f, sum->remainder);
118 }
119
120 /*
121  * set (initialize) the size entries in the per-file sum_struct
122  * calculating dynamic block and checksum sizes.
123  *
124  * This is only called from generate_and_send_sums() but is a separate
125  * function to encapsulate the logic.
126  *
127  * The block size is a rounded square root of file length.
128  *
129  * The checksum size is determined according to:
130  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
131  * provided by Donovan Baarda which gives a probability of rsync
132  * algorithm corrupting data and falling back using the whole md4
133  * checksums.
134  *
135  * This might be made one of several selectable heuristics.
136  */
137 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
138 {
139         int32 blength;
140         int s2length;
141
142         if (block_size)
143                 blength = block_size;
144         else if (len <= BLOCK_SIZE * BLOCK_SIZE)
145                 blength = BLOCK_SIZE;
146         else {
147                 int32 c;
148                 uint64 l;
149                 for (c = 1, l = len; l >>= 2; c <<= 1) {
150                     assert(c > 0);
151                 }
152                 blength = 0;
153                 do {
154                         blength |= c;
155                         if (len < (uint64)blength * blength)
156                                 blength &= ~c;
157                         c >>= 1;
158                 } while (c >= 8);       /* round to multiple of 8 */
159                 blength = MAX(blength, BLOCK_SIZE);
160         }
161
162         if (protocol_version < 27) {
163                 s2length = csum_length;
164         } else if (csum_length == SUM_LENGTH) {
165                 s2length = SUM_LENGTH;
166         } else {
167                 int32 c;
168                 uint64 l;
169                 int b = BLOCKSUM_BIAS;
170                 for (l = len; l >>= 1; b += 2) {}
171                 for (c = blength; c >>= 1 && b; b--) {}
172                 /* add a bit, subtract rollsum, round up. */
173                 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
174                 s2length = MAX(s2length, csum_length);
175                 s2length = MIN(s2length, SUM_LENGTH);
176         }
177
178         sum->flength    = len;
179         sum->blength    = blength;
180         sum->s2length   = s2length;
181         sum->count      = (len + (blength - 1)) / blength;
182         sum->remainder  = (len % blength);
183
184         if (sum->count && verbose > 2) {
185                 rprintf(FINFO,
186                         "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
187                         (double)sum->count, (long)sum->remainder, (long)sum->blength,
188                         sum->s2length, (double)sum->flength);
189         }
190 }
191
192
193 /*
194  * Generate and send a stream of signatures/checksums that describe a buffer
195  *
196  * Generate approximately one checksum every block_len bytes.
197  */
198 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
199 {
200         size_t i;
201         struct map_struct *mapbuf;
202         struct sum_struct sum;
203         OFF_T offset = 0;
204
205         sum_sizes_sqroot(&sum, len);
206
207         if (len > 0)
208                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
209         else
210                 mapbuf = NULL;
211
212         write_sum_head(f_out, &sum);
213
214         for (i = 0; i < sum.count; i++) {
215                 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
216                 char *map = map_ptr(mapbuf, offset, n1);
217                 uint32 sum1 = get_checksum1(map, n1);
218                 char sum2[SUM_LENGTH];
219
220                 if (f_copy >= 0)
221                         full_write(f_copy, map, n1);
222
223                 get_checksum2(map, n1, sum2);
224
225                 if (verbose > 3) {
226                         rprintf(FINFO,
227                                 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
228                                 (double)i, (double)offset, (long)n1,
229                                 (unsigned long)sum1);
230                 }
231                 write_int(f_out, sum1);
232                 write_buf(f_out, sum2, sum.s2length);
233                 len -= n1;
234                 offset += n1;
235         }
236
237         if (mapbuf)
238                 unmap_file(mapbuf);
239 }
240
241
242
243 /*
244  * Acts on file number @p i from @p flist, whose name is @p fname.
245  *
246  * First fixes up permissions, then generates checksums for the file.
247  *
248  * @note This comment was added later by mbp who was trying to work it
249  * out.  It might be wrong.
250  */
251 static void recv_generator(char *fname, struct file_struct *file, int i,
252                            int f_out, int f_out_name)
253 {
254         int fd = -1, f_copy = -1;
255         STRUCT_STAT st, partial_st;
256         struct file_struct *back_file = NULL;
257         int statret, stat_errno;
258         char *fnamecmp, *partialptr, *backupptr = NULL;
259         char fnamecmpbuf[MAXPATHLEN];
260         uchar fnamecmp_type;
261
262         if (list_only)
263                 return;
264
265         if (verbose > 2)
266                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
267
268         if (server_exclude_list.head
269             && check_exclude(&server_exclude_list, fname,
270                              S_ISDIR(file->mode)) < 0) {
271                 if (verbose) {
272                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
273                                 safe_fname(fname));
274                 }
275                 return;
276         }
277
278         if (dry_run > 1) {
279                 statret = -1;
280                 stat_errno = ENOENT;
281         } else {
282                 statret = link_stat(fname, &st,
283                                     keep_dirlinks && S_ISDIR(file->mode));
284                 stat_errno = errno;
285         }
286
287         if (only_existing && statret == -1 && stat_errno == ENOENT) {
288                 /* we only want to update existing files */
289                 if (verbose > 1) {
290                         rprintf(FINFO, "not creating new file \"%s\"\n",
291                                 safe_fname(fname));
292                 }
293                 return;
294         }
295
296         if (statret == 0 && !preserve_perms
297             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
298                 /* if the file exists already and we aren't perserving
299                  * permissions then act as though the remote end sent
300                  * us the file permissions we already have */
301                 file->mode = (file->mode & ~CHMOD_BITS)
302                            | (st.st_mode & CHMOD_BITS);
303         }
304
305         if (S_ISDIR(file->mode)) {
306                 /* The file to be received is a directory, so we need
307                  * to prepare appropriately.  If there is already a
308                  * file of that name and it is *not* a directory, then
309                  * we need to delete it.  If it doesn't exist, then
310                  * recursively create it. */
311
312                 if (dry_run)
313                         return; /* TODO: causes inaccuracies -- fix */
314                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
315                         if (robust_unlink(fname) != 0) {
316                                 rsyserr(FERROR, errno,
317                                         "recv_generator: unlink %s to make room for directory",
318                                         full_fname(fname));
319                                 return;
320                         }
321                         statret = -1;
322                 }
323                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
324                         if (!(relative_paths && errno == ENOENT
325                             && create_directory_path(fname, orig_umask) == 0
326                             && do_mkdir(fname, file->mode) == 0)) {
327                                 rsyserr(FERROR, errno,
328                                         "recv_generator: mkdir %s failed",
329                                         full_fname(fname));
330                         }
331                 }
332                 /* f_out is set to -1 when doing final directory-permission
333                  * and modification-time repair. */
334                 if (set_perms(fname, file, statret ? NULL : &st, 0)
335                     && verbose && f_out != -1)
336                         rprintf(FINFO, "%s/\n", safe_fname(fname));
337                 return;
338         } else if (max_size && file->length > max_size) {
339                 if (verbose > 1)
340                         rprintf(FINFO, "%s is over max-size\n", fname);
341                 return;
342         }
343
344         if (preserve_links && S_ISLNK(file->mode)) {
345 #if SUPPORT_LINKS
346                 char lnk[MAXPATHLEN];
347                 int l;
348
349                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
350                         if (verbose) {
351                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
352                                         full_fname(fname), file->u.link);
353                         }
354                         return;
355                 }
356                 if (statret == 0) {
357                         l = readlink(fname,lnk,MAXPATHLEN-1);
358                         if (l > 0) {
359                                 lnk[l] = 0;
360                                 /* A link already pointing to the
361                                  * right place -- no further action
362                                  * required. */
363                                 if (strcmp(lnk,file->u.link) == 0) {
364                                         set_perms(fname, file, &st,
365                                                   PERMS_REPORT);
366                                         return;
367                                 }
368                         }
369                         /* Not a symlink, so delete whatever's
370                          * already there and put a new symlink
371                          * in place. */
372                         delete_file(fname);
373                 }
374                 if (do_symlink(file->u.link,fname) != 0) {
375                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
376                                 full_fname(fname), safe_fname(file->u.link));
377                 } else {
378                         set_perms(fname,file,NULL,0);
379                         if (verbose) {
380                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
381                                         safe_fname(file->u.link));
382                         }
383                 }
384 #endif
385                 return;
386         }
387
388         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
389                 if (statret != 0 ||
390                     st.st_mode != file->mode ||
391                     st.st_rdev != file->u.rdev) {
392                         delete_file(fname);
393                         if (verbose > 2) {
394                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
395                                         safe_fname(fname),
396                                         (int)file->mode, (int)file->u.rdev);
397                         }
398                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
399                                 rsyserr(FERROR, errno, "mknod %s failed",
400                                         full_fname(fname));
401                         } else {
402                                 set_perms(fname,file,NULL,0);
403                                 if (verbose) {
404                                         rprintf(FINFO, "%s\n",
405                                                 safe_fname(fname));
406                                 }
407                         }
408                 } else {
409                         set_perms(fname, file, &st, PERMS_REPORT);
410                 }
411                 return;
412         }
413
414         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
415                 return;
416
417         if (!S_ISREG(file->mode)) {
418                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
419                         safe_fname(fname));
420                 return;
421         }
422
423         fnamecmp = fname;
424         fnamecmp_type = FNAMECMP_FNAME;
425
426         if (statret == -1 && basis_dir[0] != NULL) {
427                 int fallback_match = -1;
428                 int match_level = 0;
429                 int i = 0;
430                 do {
431                         pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
432                                  basis_dir[i], fname);
433                         if (link_stat(fnamecmpbuf, &st, 0) == 0
434                             && S_ISREG(st.st_mode)) {
435                                 statret = 0;
436                                 if (link_dest) {
437                                         if (!match_level) {
438                                                 fallback_match = i;
439                                                 match_level = 1;
440                                         } else if (match_level == 2
441                                             && !unchanged_attrs(file, &st))
442                                                 continue;
443                                         if (!unchanged_file(fnamecmpbuf, file, &st))
444                                                 continue;
445                                         fallback_match = i;
446                                         match_level = 2;
447                                         if (!unchanged_attrs(file, &st))
448                                                 continue;
449                                 }
450                                 match_level = 3;
451                                 break;
452                         }
453                 } while (basis_dir[++i] != NULL);
454                 if (statret == 0) {
455                         if (match_level < 3) {
456                                 i = fallback_match;
457                                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
458                                          basis_dir[i], fname);
459                         }
460 #if HAVE_LINK
461                         if (link_dest && match_level == 3 && !dry_run) {
462                                 if (do_link(fnamecmpbuf, fname) < 0) {
463                                         if (verbose) {
464                                                 rsyserr(FINFO, errno,
465                                                         "link %s => %s",
466                                                         fnamecmpbuf,
467                                                         safe_fname(fname));
468                                         }
469                                         fnamecmp = fnamecmpbuf;
470                                         fnamecmp_type = FNAMECMP_BASIS_DIR + i;
471                                 }
472                         } else
473 #endif
474                         {
475                                 fnamecmp = fnamecmpbuf;
476                                 fnamecmp_type = FNAMECMP_BASIS_DIR + i;
477                         }
478                 }
479         }
480
481         if (statret == 0 && !S_ISREG(st.st_mode)) {
482                 if (delete_file(fname) != 0)
483                         return;
484                 statret = -1;
485                 stat_errno = ENOENT;
486         }
487
488         if (partial_dir && (partialptr = partial_dir_fname(fname))
489             && link_stat(partialptr, &partial_st, 0) == 0
490             && S_ISREG(partial_st.st_mode)) {
491                 if (statret == -1)
492                         goto prepare_to_open;
493         } else
494                 partialptr = NULL;
495
496         if (statret == -1) {
497                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
498                         return;
499                 if (stat_errno == ENOENT)
500                         goto notify_others;
501                 if (verbose > 1) {
502                         rsyserr(FERROR, stat_errno,
503                                 "recv_generator: failed to stat %s",
504                                 full_fname(fname));
505                 }
506                 return;
507         }
508
509         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
510                 if (verbose > 1)
511                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
512                 return;
513         }
514
515         if (update_only && fnamecmp_type == FNAMECMP_FNAME
516             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
517                 if (verbose > 1)
518                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
519                 return;
520         }
521
522         if ((link_dest || copy_dest) && fnamecmp_type != FNAMECMP_FNAME)
523                 ;
524         else if (unchanged_file(fnamecmp, file, &st)) {
525                 if (fnamecmp_type == FNAMECMP_FNAME)
526                         set_perms(fname, file, &st, PERMS_REPORT);
527                 return;
528         }
529
530 prepare_to_open:
531         if (dry_run || whole_file > 0) {
532                 statret = -1;
533                 goto notify_others;
534         }
535         if (read_batch)
536                 goto notify_others;
537
538         if (partialptr) {
539                 st = partial_st;
540                 fnamecmp = partialptr;
541                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
542         }
543
544         /* open the file */
545         fd = do_open(fnamecmp, O_RDONLY, 0);
546
547         if (fd == -1) {
548                 rsyserr(FERROR, errno, "failed to open %s, continuing",
549                         full_fname(fnamecmp));
550             pretend_missing:
551                 /* pretend the file didn't exist */
552                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
553                         return;
554                 statret = -1;
555                 goto notify_others;
556         }
557
558         if (inplace && make_backups) {
559                 if (!(backupptr = get_backup_name(fname))) {
560                         close(fd);
561                         return;
562                 }
563                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
564                         close(fd);
565                         goto pretend_missing;
566                 }
567                 if (robust_unlink(backupptr) && errno != ENOENT) {
568                         rsyserr(FERROR, errno, "unlink %s",
569                                 full_fname(backupptr));
570                         free(back_file);
571                         close(fd);
572                         return;
573                 }
574                 if ((f_copy = do_open(backupptr,
575                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
576                         rsyserr(FERROR, errno, "open %s",
577                                 full_fname(backupptr));
578                         free(back_file);
579                         close(fd);
580                         return;
581                 }
582                 fnamecmp_type = FNAMECMP_BACKUP;
583         }
584
585         if (verbose > 3) {
586                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
587                         safe_fname(fnamecmp), (double)st.st_size);
588         }
589
590         if (verbose > 2)
591                 rprintf(FINFO, "generating and sending sums for %d\n", i);
592
593 notify_others:
594         write_int(f_out, i);
595         if (f_out_name >= 0)
596                 write_byte(f_out_name, fnamecmp_type);
597
598         if (dry_run || read_batch)
599                 return;
600
601         if (statret == 0) {
602                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
603
604                 if (f_copy >= 0) {
605                         close(f_copy);
606                         set_perms(backupptr, back_file, NULL, 0);
607                         if (verbose > 1) {
608                                 rprintf(FINFO, "backed up %s to %s\n",
609                                         fname, backupptr);
610                         }
611                         free(back_file);
612                 }
613
614                 close(fd);
615         } else
616                 write_sum_head(f_out, NULL);
617 }
618
619
620 void generate_files(int f_out, struct file_list *flist, char *local_name,
621                     int f_out_name)
622 {
623         int i;
624         int phase = 0;
625         char fbuf[MAXPATHLEN];
626
627         if (verbose > 2) {
628                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
629                         (long)getpid(), flist->count);
630         }
631
632         if (verbose >= 2) {
633                 rprintf(FINFO,
634                         whole_file > 0
635                         ? "delta-transmission disabled for local transfer or --whole-file\n"
636                         : "delta transmission enabled\n");
637         }
638
639         /* we expect to just sit around now, so don't exit on a
640            timeout. If we really get a timeout then the other process should
641            exit */
642         io_timeout = 0;
643
644         for (i = 0; i < flist->count; i++) {
645                 struct file_struct *file = flist->files[i];
646                 struct file_struct copy;
647
648                 if (!file->basename)
649                         continue;
650                 /* we need to ensure that any directories we create have writeable
651                    permissions initially so that we can create the files within
652                    them. This is then fixed after the files are transferred */
653                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
654                         copy = *file;
655                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
656                          * handling of permissions is strange? */
657                         copy.mode |= S_IWUSR; /* user write */
658                         file = &copy;
659                 }
660
661                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
662                                file, i, f_out, f_out_name);
663         }
664
665         phase++;
666         csum_length = SUM_LENGTH;
667         ignore_times = 1;
668
669         if (verbose > 2)
670                 rprintf(FINFO,"generate_files phase=%d\n",phase);
671
672         write_int(f_out, -1);
673
674         /* files can cycle through the system more than once
675          * to catch initial checksum errors */
676         while ((i = get_redo_num()) != -1) {
677                 struct file_struct *file = flist->files[i];
678                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
679                                file, i, f_out, f_out_name);
680         }
681
682         phase++;
683         if (verbose > 2)
684                 rprintf(FINFO,"generate_files phase=%d\n",phase);
685
686         write_int(f_out, -1);
687
688         /* Read post-redo-phase MSG_DONE and any prior messages. */
689         get_redo_num();
690
691         if (preserve_hard_links)
692                 do_hard_links();
693
694         /* now we need to fix any directory permissions that were
695          * modified during the transfer */
696         for (i = 0; i < flist->count; i++) {
697                 struct file_struct *file = flist->files[i];
698                 if (!file->basename || !S_ISDIR(file->mode))
699                         continue;
700                 recv_generator(local_name ? local_name : f_name(file),
701                                file, i, -1, -1);
702         }
703
704         if (verbose > 2)
705                 rprintf(FINFO,"generate_files finished\n");
706 }