1 /* -*- c-file-style: "linux" -*-
3 rsync -- fast file replication program
5 Copyright (C) 1996-2000 by Andrew Tridgell
6 Copyright (C) Paul Mackerras 1996
7 Copyright (C) 2002 by Martin Pool <mbp@samba.org>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int preserve_times;
38 extern int omit_dir_times;
39 extern int delete_during;
40 extern int update_only;
41 extern int opt_ignore_existing;
43 extern int make_backups;
44 extern int csum_length;
45 extern int ignore_times;
47 extern OFF_T max_size;
48 extern int io_timeout;
49 extern int protocol_version;
50 extern int always_checksum;
51 extern char *partial_dir;
52 extern char *basis_dir[];
53 extern int compare_dest;
55 extern int whole_file;
56 extern int local_server;
58 extern int read_batch;
59 extern int only_existing;
60 extern int orig_umask;
61 extern int safe_symlinks;
62 extern long block_size; /* "long" because popt can't set an int32. */
64 extern struct filter_list_struct server_filter_list;
66 static int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
69 && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
72 if (am_root && preserve_uid && st->st_uid != file->uid)
75 if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
81 /* Perform our quick-check heuristic for determining if a file is unchanged. */
82 static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
84 if (st->st_size != file->length)
87 /* if always checksum is set then we use the checksum instead
88 of the file time to determine whether to sync */
89 if (always_checksum && S_ISREG(st->st_mode)) {
90 char sum[MD4_SUM_LENGTH];
91 file_checksum(fn, sum, st->st_size);
92 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
93 : MD4_SUM_LENGTH) == 0;
102 return cmp_modtime(st->st_mtime, file->modtime) == 0;
107 * set (initialize) the size entries in the per-file sum_struct
108 * calculating dynamic block and checksum sizes.
110 * This is only called from generate_and_send_sums() but is a separate
111 * function to encapsulate the logic.
113 * The block size is a rounded square root of file length.
115 * The checksum size is determined according to:
116 * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
117 * provided by Donovan Baarda which gives a probability of rsync
118 * algorithm corrupting data and falling back using the whole md4
121 * This might be made one of several selectable heuristics.
123 static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
129 blength = block_size;
130 else if (len <= BLOCK_SIZE * BLOCK_SIZE)
131 blength = BLOCK_SIZE;
136 for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
137 if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
138 blength = MAX_BLOCK_SIZE;
143 if (len < (int64)blength * blength)
146 } while (c >= 8); /* round to multiple of 8 */
147 blength = MAX(blength, BLOCK_SIZE);
151 if (protocol_version < 27) {
152 s2length = csum_length;
153 } else if (csum_length == SUM_LENGTH) {
154 s2length = SUM_LENGTH;
158 int b = BLOCKSUM_BIAS;
159 for (l = len; l >>= 1; b += 2) {}
160 for (c = blength; c >>= 1 && b; b--) {}
161 /* add a bit, subtract rollsum, round up. */
162 s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
163 s2length = MAX(s2length, csum_length);
164 s2length = MIN(s2length, SUM_LENGTH);
168 sum->blength = blength;
169 sum->s2length = s2length;
170 sum->count = (len + (blength - 1)) / blength;
171 sum->remainder = (len % blength);
173 if (sum->count && verbose > 2) {
175 "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
176 (double)sum->count, (long)sum->remainder, (long)sum->blength,
177 sum->s2length, (double)sum->flength);
183 * Generate and send a stream of signatures/checksums that describe a buffer
185 * Generate approximately one checksum every block_len bytes.
187 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
190 struct map_struct *mapbuf;
191 struct sum_struct sum;
194 sum_sizes_sqroot(&sum, len);
197 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
201 write_sum_head(f_out, &sum);
203 for (i = 0; i < sum.count; i++) {
204 int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
205 char *map = map_ptr(mapbuf, offset, n1);
206 uint32 sum1 = get_checksum1(map, n1);
207 char sum2[SUM_LENGTH];
210 full_write(f_copy, map, n1);
212 get_checksum2(map, n1, sum2);
216 "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
217 (double)i, (double)offset, (long)n1,
218 (unsigned long)sum1);
220 write_int(f_out, sum1);
221 write_buf(f_out, sum2, sum.s2length);
232 * Acts on file number @p i from @p flist, whose name is @p fname.
234 * First fixes up permissions, then generates checksums for the file.
236 * @note This comment was added later by mbp who was trying to work it
237 * out. It might be wrong.
239 static void recv_generator(char *fname, struct file_list *flist,
240 struct file_struct *file, int i,
241 int f_out, int f_out_name)
243 int fd = -1, f_copy = -1;
244 STRUCT_STAT st, partial_st;
245 struct file_struct *back_file = NULL;
246 int statret, stat_errno;
247 char *fnamecmp, *partialptr, *backupptr = NULL;
248 char fnamecmpbuf[MAXPATHLEN];
255 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
257 if (server_filter_list.head
258 && check_filter(&server_filter_list, fname,
259 S_ISDIR(file->mode)) < 0) {
261 rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
271 statret = link_stat(fname, &st,
272 keep_dirlinks && S_ISDIR(file->mode));
276 if (only_existing && statret == -1 && stat_errno == ENOENT) {
277 /* we only want to update existing files */
279 rprintf(FINFO, "not creating new file \"%s\"\n",
285 if (statret == 0 && !preserve_perms
286 && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
287 /* if the file exists already and we aren't perserving
288 * permissions then act as though the remote end sent
289 * us the file permissions we already have */
290 file->mode = (file->mode & ~CHMOD_BITS)
291 | (st.st_mode & CHMOD_BITS);
294 if (S_ISDIR(file->mode)) {
295 /* The file to be received is a directory, so we need
296 * to prepare appropriately. If there is already a
297 * file of that name and it is *not* a directory, then
298 * we need to delete it. If it doesn't exist, then
299 * (perhaps recursively) create it. */
300 if (statret == 0 && !S_ISDIR(st.st_mode)) {
301 delete_file(fname, DEL_TERSE);
304 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
305 if (!relative_paths || errno != ENOENT
306 || create_directory_path(fname, orig_umask) < 0
307 || do_mkdir(fname, file->mode) < 0) {
308 rsyserr(FERROR, errno,
309 "recv_generator: mkdir %s failed",
313 /* f_out is set to -1 when doing final directory-permission
314 * and modification-time repair. */
315 if (set_perms(fname, file, statret ? NULL : &st, 0)
316 && verbose && f_out != -1)
317 rprintf(FINFO, "%s/\n", safe_fname(fname));
318 if (delete_during && f_out != -1 && csum_length != SUM_LENGTH
319 && (file->flags & FLAG_DEL_START))
320 delete_in_dir(flist, fname);
322 } else if (max_size && file->length > max_size) {
324 rprintf(FINFO, "%s is over max-size\n", fname);
328 if (preserve_links && S_ISLNK(file->mode)) {
330 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
332 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
333 full_fname(fname), file->u.link);
338 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
339 char lnk[MAXPATHLEN];
343 && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
345 /* A link already pointing to the
346 * right place -- no further action
348 if (strcmp(lnk, file->u.link) == 0) {
349 set_perms(fname, file, &st,
354 /* Not the right symlink (or not a symlink), so
356 delete_file(fname, dflag | DEL_TERSE);
358 if (do_symlink(file->u.link,fname) != 0) {
359 rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
360 full_fname(fname), safe_fname(file->u.link));
362 set_perms(fname,file,NULL,0);
364 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
365 safe_fname(file->u.link));
372 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
374 st.st_mode != file->mode ||
375 st.st_rdev != file->u.rdev) {
376 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
377 delete_file(fname, dflag | DEL_TERSE);
379 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
381 (int)file->mode, (int)file->u.rdev);
383 if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
384 rsyserr(FERROR, errno, "mknod %s failed",
387 set_perms(fname,file,NULL,0);
389 rprintf(FINFO, "%s\n",
394 set_perms(fname, file, &st, PERMS_REPORT);
399 if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
402 if (!S_ISREG(file->mode)) {
403 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
409 fnamecmp_type = FNAMECMP_FNAME;
411 if (statret == -1 && basis_dir[0] != NULL) {
412 int fallback_match = -1;
416 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
417 basis_dir[i], fname);
418 if (link_stat(fnamecmpbuf, &st, 0) == 0
419 && S_ISREG(st.st_mode)) {
425 } else if (match_level == 2
426 && !unchanged_attrs(file, &st))
428 if (!unchanged_file(fnamecmpbuf, file, &st))
432 if (!unchanged_attrs(file, &st))
438 } while (basis_dir[++i] != NULL);
440 if (match_level < 3) {
442 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
443 basis_dir[i], fname);
446 if (link_dest && match_level == 3 && !dry_run) {
447 if (do_link(fnamecmpbuf, fname) < 0) {
449 rsyserr(FINFO, errno,
454 fnamecmp = fnamecmpbuf;
460 fnamecmp = fnamecmpbuf;
466 if (statret == 0 && !S_ISREG(st.st_mode)) {
467 int dflag = S_ISDIR(st.st_mode) ? DEL_DIR : 0;
468 if (delete_file(fname, dflag | DEL_TERSE) != 0)
474 if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
475 && link_stat(partialptr, &partial_st, 0) == 0
476 && S_ISREG(partial_st.st_mode)) {
478 goto prepare_to_open;
483 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
485 if (stat_errno == ENOENT)
488 rsyserr(FERROR, stat_errno,
489 "recv_generator: failed to stat %s",
495 if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
497 rprintf(FINFO, "%s exists\n", safe_fname(fname));
501 if (update_only && fnamecmp_type == FNAMECMP_FNAME
502 && cmp_modtime(st.st_mtime, file->modtime) > 0) {
504 rprintf(FINFO, "%s is newer\n", safe_fname(fname));
508 if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
510 else if (unchanged_file(fnamecmp, file, &st)) {
511 if (fnamecmp_type == FNAMECMP_FNAME)
512 set_perms(fname, file, &st, PERMS_REPORT);
519 fnamecmp = partialptr;
520 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
524 if (dry_run || whole_file > 0) {
532 fd = do_open(fnamecmp, O_RDONLY, 0);
535 rsyserr(FERROR, errno, "failed to open %s, continuing",
536 full_fname(fnamecmp));
538 /* pretend the file didn't exist */
539 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
545 if (inplace && make_backups) {
546 if (!(backupptr = get_backup_name(fname))) {
550 if (!(back_file = make_file(fname, NULL, NO_FILTERS))) {
552 goto pretend_missing;
554 if (robust_unlink(backupptr) && errno != ENOENT) {
555 rsyserr(FERROR, errno, "unlink %s",
556 full_fname(backupptr));
561 if ((f_copy = do_open(backupptr,
562 O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
563 rsyserr(FERROR, errno, "open %s",
564 full_fname(backupptr));
569 fnamecmp_type = FNAMECMP_BACKUP;
573 rprintf(FINFO, "gen mapped %s of size %.0f\n",
574 safe_fname(fnamecmp), (double)st.st_size);
578 rprintf(FINFO, "generating and sending sums for %d\n", i);
582 if (protocol_version >= 29 && inplace && !read_batch)
583 write_byte(f_out, fnamecmp_type);
585 write_byte(f_out_name, fnamecmp_type);
587 if (dry_run || read_batch)
591 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
595 set_perms(backupptr, back_file, NULL, 0);
597 rprintf(FINFO, "backed up %s to %s\n",
605 write_sum_head(f_out, NULL);
609 void generate_files(int f_out, struct file_list *flist, char *local_name,
614 char fbuf[MAXPATHLEN];
615 int need_retouch_dir_times = preserve_times && !omit_dir_times;
616 int need_retouch_dir_perms = 0;
619 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
620 (long)getpid(), flist->count);
626 ? "delta-transmission disabled for local transfer or --whole-file\n"
627 : "delta transmission enabled\n");
630 /* we expect to just sit around now, so don't exit on a
631 timeout. If we really get a timeout then the other process should
635 for (i = 0; i < flist->count; i++) {
636 struct file_struct *file = flist->files[i];
637 struct file_struct copy;
641 /* we need to ensure that any directories we create have writeable
642 permissions initially so that we can create the files within
643 them. This is then fixed after the files are transferred */
644 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
646 copy.mode |= S_IWUSR; /* user write */
648 need_retouch_dir_perms = 1;
651 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
652 flist, file, i, f_out, f_out_name);
655 delete_in_dir(NULL, NULL);
658 csum_length = SUM_LENGTH;
662 rprintf(FINFO,"generate_files phase=%d\n",phase);
664 write_int(f_out, -1);
666 /* files can cycle through the system more than once
667 * to catch initial checksum errors */
668 while ((i = get_redo_num()) != -1) {
669 struct file_struct *file = flist->files[i];
670 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
671 flist, file, i, f_out, f_out_name);
676 rprintf(FINFO,"generate_files phase=%d\n",phase);
678 write_int(f_out, -1);
680 /* Read post-redo-phase MSG_DONE and any prior messages. */
683 if (preserve_hard_links)
686 if (need_retouch_dir_perms || need_retouch_dir_times) {
687 /* Now we need to fix any directory permissions that were
688 * modified during the transfer and/or re-set any tweaked
689 * modified-time values. */
690 for (i = 0; i < flist->count; i++) {
691 struct file_struct *file = flist->files[i];
692 if (!file->basename || !S_ISDIR(file->mode))
694 if (!need_retouch_dir_times && file->mode & S_IWUSR)
696 recv_generator(local_name ? local_name : f_name(file),
697 flist, file, i, -1, -1);
702 rprintf(FINFO,"generate_files finished\n");