1 /* -*- c-file-style: "linux" -*-
3 rsync -- fast file replication program
5 Copyright (C) 1996-2000 by Andrew Tridgell
6 Copyright (C) Paul Mackerras 1996
7 Copyright (C) 2002 by Martin Pool <mbp@samba.org>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int csum_length;
40 extern int ignore_times;
42 extern int io_timeout;
43 extern int protocol_version;
44 extern int always_checksum;
45 extern char *partial_dir;
46 extern char *compare_dest;
48 extern int whole_file;
49 extern int local_server;
51 extern int read_batch;
52 extern int only_existing;
53 extern int orig_umask;
54 extern int safe_symlinks;
55 extern unsigned int block_size;
57 extern struct exclude_list_struct server_exclude_list;
60 /* choose whether to skip a particular file */
61 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
63 if (st->st_size != file->length)
67 && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
70 if (am_root && preserve_uid && st->st_uid != file->uid)
73 if (preserve_gid && file->gid != GID_NONE
74 && st->st_gid != file->gid)
78 /* if always checksum is set then we use the checksum instead
79 of the file time to determine whether to sync */
80 if (always_checksum && S_ISREG(st->st_mode)) {
81 char sum[MD4_SUM_LENGTH];
82 file_checksum(fname,sum,st->st_size);
83 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
84 : MD4_SUM_LENGTH) == 0;
93 return cmp_modtime(st->st_mtime, file->modtime) == 0;
98 * NULL sum_struct means we have no checksums
100 void write_sum_head(int f, struct sum_struct *sum)
102 static struct sum_struct null_sum;
107 write_int(f, sum->count);
108 write_int(f, sum->blength);
109 if (protocol_version >= 27)
110 write_int(f, sum->s2length);
111 write_int(f, sum->remainder);
115 * set (initialize) the size entries in the per-file sum_struct
116 * calculating dynamic block and checksum sizes.
118 * This is only called from generate_and_send_sums() but is a separate
119 * function to encapsulate the logic.
121 * The block size is a rounded square root of file length.
123 * The checksum size is determined according to:
124 * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
125 * provided by Donovan Baarda which gives a probability of rsync
126 * algorithm corrupting data and falling back using the whole md4
129 * This might be made one of several selectable heuristics.
132 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
134 unsigned int blength;
140 blength = block_size;
141 } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
142 blength = BLOCK_SIZE;
152 if (len < (uint64)blength * blength)
155 } while (c >= 8); /* round to multiple of 8 */
156 blength = MAX(blength, BLOCK_SIZE);
159 if (protocol_version < 27) {
160 s2length = csum_length;
161 } else if (csum_length == SUM_LENGTH) {
162 s2length = SUM_LENGTH;
164 int b = BLOCKSUM_BIAS;
170 while (c >>= 1 && b) {
173 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
176 * --optimize in compiler--
178 s2length = MAX(s2length, csum_length);
179 s2length = MIN(s2length, SUM_LENGTH);
183 sum->blength = blength;
184 sum->s2length = s2length;
185 sum->count = (len + (blength - 1)) / blength;
186 sum->remainder = (len % blength);
188 if (sum->count && verbose > 2) {
189 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
190 (double)sum->count, sum->remainder, sum->blength,
191 sum->s2length, (double)sum->flength);
197 * Generate and send a stream of signatures/checksums that describe a buffer
199 * Generate approximately one checksum every block_len bytes.
201 static void generate_and_send_sums(int fd, OFF_T len, int f_out)
204 struct map_struct *mapbuf;
205 struct sum_struct sum;
208 sum_sizes_sqroot(&sum, len);
211 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
215 write_sum_head(f_out, &sum);
217 for (i = 0; i < sum.count; i++) {
218 unsigned int n1 = MIN(len, sum.blength);
219 char *map = map_ptr(mapbuf, offset, n1);
220 uint32 sum1 = get_checksum1(map, n1);
221 char sum2[SUM_LENGTH];
223 get_checksum2(map, n1, sum2);
227 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
228 (double)i, (double)offset, n1,
229 (unsigned long)sum1);
231 write_int(f_out, sum1);
232 write_buf(f_out, sum2, sum.s2length);
244 * Acts on file number @p i from @p flist, whose name is @p fname.
246 * First fixes up permissions, then generates checksums for the file.
248 * @note This comment was added later by mbp who was trying to work it
249 * out. It might be wrong.
251 static void recv_generator(char *fname, struct file_struct *file, int i,
256 int statret, stat_errno;
258 char fnamecmpbuf[MAXPATHLEN];
264 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
266 if (server_exclude_list.head
267 && check_exclude(&server_exclude_list, fname,
268 S_ISDIR(file->mode)) < 0) {
270 rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
276 statret = link_stat(fname, &st, keep_dirlinks && S_ISDIR(file->mode));
279 if (only_existing && statret == -1 && stat_errno == ENOENT) {
280 /* we only want to update existing files */
282 rprintf(FINFO, "not creating new file \"%s\"\n",
288 if (statret == 0 && !preserve_perms
289 && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
290 /* if the file exists already and we aren't perserving
291 * permissions then act as though the remote end sent
292 * us the file permissions we already have */
293 file->mode = (file->mode & ~CHMOD_BITS)
294 | (st.st_mode & CHMOD_BITS);
297 if (S_ISDIR(file->mode)) {
298 /* The file to be received is a directory, so we need
299 * to prepare appropriately. If there is already a
300 * file of that name and it is *not* a directory, then
301 * we need to delete it. If it doesn't exist, then
302 * recursively create it. */
305 return; /* TODO: causes inaccuracies -- fix */
306 if (statret == 0 && !S_ISDIR(st.st_mode)) {
307 if (robust_unlink(fname) != 0) {
308 rsyserr(FERROR, errno,
309 "recv_generator: unlink %s to make room for directory",
315 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
316 if (!(relative_paths && errno == ENOENT
317 && create_directory_path(fname, orig_umask) == 0
318 && do_mkdir(fname, file->mode) == 0)) {
319 rsyserr(FERROR, errno,
320 "recv_generator: mkdir %s failed",
324 /* f_out is set to -1 when doing final directory-permission
325 * and modification-time repair. */
326 if (set_perms(fname, file, statret ? NULL : &st, 0)
327 && verbose && f_out != -1)
328 rprintf(FINFO, "%s/\n", safe_fname(fname));
332 if (preserve_links && S_ISLNK(file->mode)) {
334 char lnk[MAXPATHLEN];
337 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
339 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
340 full_fname(fname), file->u.link);
345 l = readlink(fname,lnk,MAXPATHLEN-1);
348 /* A link already pointing to the
349 * right place -- no further action
351 if (strcmp(lnk,file->u.link) == 0) {
352 set_perms(fname, file, &st,
357 /* Not a symlink, so delete whatever's
358 * already there and put a new symlink
362 if (do_symlink(file->u.link,fname) != 0) {
363 rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
364 full_fname(fname), safe_fname(file->u.link));
366 set_perms(fname,file,NULL,0);
368 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
369 safe_fname(file->u.link));
377 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
379 st.st_mode != file->mode ||
380 st.st_rdev != file->u.rdev) {
383 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
385 (int)file->mode, (int)file->u.rdev);
387 if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
388 rsyserr(FERROR, errno, "mknod %s failed",
391 set_perms(fname,file,NULL,0);
393 rprintf(FINFO, "%s\n",
398 set_perms(fname, file, &st, PERMS_REPORT);
404 if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
407 if (!S_ISREG(file->mode)) {
408 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
415 if (statret == -1 && compare_dest != NULL) {
416 /* try the file at compare_dest instead */
417 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
418 if (link_stat(fnamecmpbuf, &st, 0) == 0
419 && S_ISREG(st.st_mode)) {
421 if (link_dest && !dry_run) {
422 if (do_link(fnamecmpbuf, fname) < 0) {
424 rsyserr(FINFO, errno,
429 fnamecmp = fnamecmpbuf;
433 fnamecmp = fnamecmpbuf;
438 if (statret == 0 && !S_ISREG(st.st_mode)) {
439 if (delete_file(fname) != 0)
446 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
448 if (stat_errno == ENOENT) {
450 if (!dry_run && !read_batch)
451 write_sum_head(f_out, NULL);
452 } else if (verbose > 1) {
453 rsyserr(FERROR, stat_errno,
454 "recv_generator: failed to stat %s",
460 if (opt_ignore_existing && fnamecmp == fname) {
462 rprintf(FINFO, "%s exists\n", safe_fname(fname));
466 if (update_only && fnamecmp == fname
467 && cmp_modtime(st.st_mtime, file->modtime) > 0) {
469 rprintf(FINFO, "%s is newer\n", safe_fname(fname));
473 if (skip_file(fnamecmp, file, &st)) {
474 if (fnamecmp == fname)
475 set_perms(fname, file, &st, PERMS_REPORT);
479 if (dry_run || read_batch) {
484 if (whole_file > 0) {
486 write_sum_head(f_out, NULL);
492 char *partialptr = partial_dir_fname(fname);
493 if (partialptr && link_stat(partialptr, &st2, 0) == 0
494 && S_ISREG(st2.st_mode)) {
496 fnamecmp = partialptr;
501 fd = do_open(fnamecmp, O_RDONLY, 0);
504 rsyserr(FERROR, errno, "failed to open %s, continuing",
505 full_fname(fnamecmp));
506 /* pretend the file didn't exist */
507 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
510 write_sum_head(f_out, NULL);
515 rprintf(FINFO, "gen mapped %s of size %.0f\n",
516 safe_fname(fnamecmp), (double)st.st_size);
520 rprintf(FINFO, "generating and sending sums for %d\n", i);
523 generate_and_send_sums(fd, st.st_size, f_out);
529 void generate_files(int f_out, struct file_list *flist, char *local_name)
533 char fbuf[MAXPATHLEN];
536 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
537 (long)getpid(), flist->count);
543 ? "delta-transmission disabled for local transfer or --whole-file\n"
544 : "delta transmission enabled\n");
547 /* we expect to just sit around now, so don't exit on a
548 timeout. If we really get a timeout then the other process should
552 for (i = 0; i < flist->count; i++) {
553 struct file_struct *file = flist->files[i];
554 struct file_struct copy;
558 /* we need to ensure that any directories we create have writeable
559 permissions initially so that we can create the files within
560 them. This is then fixed after the files are transferred */
561 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
563 /* XXX: Could this be causing a problem on SCO? Perhaps their
564 * handling of permissions is strange? */
565 copy.mode |= S_IWUSR; /* user write */
569 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
574 csum_length = SUM_LENGTH;
578 rprintf(FINFO,"generate_files phase=%d\n",phase);
580 write_int(f_out, -1);
582 /* files can cycle through the system more than once
583 * to catch initial checksum errors */
584 while ((i = get_redo_num()) != -1) {
585 struct file_struct *file = flist->files[i];
586 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
592 rprintf(FINFO,"generate_files phase=%d\n",phase);
594 write_int(f_out, -1);
596 if (preserve_hard_links)
599 /* now we need to fix any directory permissions that were
600 * modified during the transfer */
601 for (i = 0; i < flist->count; i++) {
602 struct file_struct *file = flist->files[i];
603 if (!file->basename || !S_ISDIR(file->mode))
605 recv_generator(local_name ? local_name : f_name(file),
610 rprintf(FINFO,"generate_files finished\n");