1 /* -*- c-file-style: "linux" -*-
3 rsync -- fast file replication program
5 Copyright (C) 1996-2000 by Andrew Tridgell
6 Copyright (C) Paul Mackerras 1996
7 Copyright (C) 2002 by Martin Pool <mbp@samba.org>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 extern int relative_paths;
29 extern int keep_dirlinks;
30 extern int preserve_links;
32 extern int preserve_devices;
33 extern int preserve_hard_links;
34 extern int preserve_perms;
35 extern int preserve_uid;
36 extern int preserve_gid;
37 extern int update_only;
38 extern int opt_ignore_existing;
39 extern int csum_length;
40 extern int ignore_times;
42 extern int io_timeout;
43 extern int protocol_version;
44 extern int always_checksum;
45 extern char *partial_dir;
46 extern char *compare_dest;
48 extern int whole_file;
49 extern int local_server;
51 extern int read_batch;
52 extern int only_existing;
53 extern int orig_umask;
54 extern int safe_symlinks;
55 extern unsigned int block_size;
57 extern struct exclude_list_struct server_exclude_list;
60 /* choose whether to skip a particular file */
61 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
63 if (st->st_size != file->length)
67 && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
70 if (am_root && preserve_uid && st->st_uid != file->uid)
73 if (preserve_gid && file->gid != GID_NONE
74 && st->st_gid != file->gid)
78 /* if always checksum is set then we use the checksum instead
79 of the file time to determine whether to sync */
80 if (always_checksum && S_ISREG(st->st_mode)) {
81 char sum[MD4_SUM_LENGTH];
82 file_checksum(fname,sum,st->st_size);
83 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
84 : MD4_SUM_LENGTH) == 0;
93 return cmp_modtime(st->st_mtime, file->modtime) == 0;
98 * NULL sum_struct means we have no checksums
100 void write_sum_head(int f, struct sum_struct *sum)
102 static struct sum_struct null_sum;
107 write_int(f, sum->count);
108 write_int(f, sum->blength);
109 if (protocol_version >= 27)
110 write_int(f, sum->s2length);
111 write_int(f, sum->remainder);
115 * set (initialize) the size entries in the per-file sum_struct
116 * calculating dynamic block and checksum sizes.
118 * This is only called from generate_and_send_sums() but is a separate
119 * function to encapsulate the logic.
121 * The block size is a rounded square root of file length.
123 * The checksum size is determined according to:
124 * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
125 * provided by Donovan Baarda which gives a probability of rsync
126 * algorithm corrupting data and falling back using the whole md4
129 * This might be made one of several selectable heuristics.
132 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
134 unsigned int blength;
140 blength = block_size;
141 } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
142 blength = BLOCK_SIZE;
152 if (len < (uint64)blength * blength)
155 } while (c >= 8); /* round to multiple of 8 */
156 blength = MAX(blength, BLOCK_SIZE);
157 blength = MIN(blength, MAX_MAP_SIZE);
160 if (protocol_version < 27) {
161 s2length = csum_length;
162 } else if (csum_length == SUM_LENGTH) {
163 s2length = SUM_LENGTH;
165 int b = BLOCKSUM_BIAS;
171 while (c >>= 1 && b) {
174 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
177 * --optimize in compiler--
179 s2length = MAX(s2length, csum_length);
180 s2length = MIN(s2length, SUM_LENGTH);
184 sum->blength = blength;
185 sum->s2length = s2length;
186 sum->count = (len + (blength - 1)) / blength;
187 sum->remainder = (len % blength);
189 if (sum->count && verbose > 2) {
190 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
191 (double)sum->count, sum->remainder, sum->blength,
192 sum->s2length, (double)sum->flength);
198 * Generate and send a stream of signatures/checksums that describe a buffer
200 * Generate approximately one checksum every block_len bytes.
202 static void generate_and_send_sums(int fd, OFF_T len, int f_out)
205 struct map_struct *mapbuf;
206 struct sum_struct sum;
209 sum_sizes_sqroot(&sum, len);
212 mapbuf = map_file(fd, len, sum.blength);
216 write_sum_head(f_out, &sum);
218 for (i = 0; i < sum.count; i++) {
219 unsigned int n1 = MIN(len, sum.blength);
220 char *map = map_ptr(mapbuf, offset, n1);
221 uint32 sum1 = get_checksum1(map, n1);
222 char sum2[SUM_LENGTH];
224 get_checksum2(map, n1, sum2);
228 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
229 (double)i, (double)offset, n1,
230 (unsigned long)sum1);
232 write_int(f_out, sum1);
233 write_buf(f_out, sum2, sum.s2length);
245 * Acts on file number @p i from @p flist, whose name is @p fname.
247 * First fixes up permissions, then generates checksums for the file.
249 * @note This comment was added later by mbp who was trying to work it
250 * out. It might be wrong.
252 static void recv_generator(char *fname, struct file_struct *file, int i,
257 int statret, stat_errno;
259 char fnamecmpbuf[MAXPATHLEN];
265 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
267 if (server_exclude_list.head
268 && check_exclude(&server_exclude_list, fname,
269 S_ISDIR(file->mode)) < 0) {
271 rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
277 statret = link_stat(fname, &st, keep_dirlinks && S_ISDIR(file->mode));
280 if (only_existing && statret == -1 && stat_errno == ENOENT) {
281 /* we only want to update existing files */
283 rprintf(FINFO, "not creating new file \"%s\"\n",
289 if (statret == 0 && !preserve_perms
290 && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
291 /* if the file exists already and we aren't perserving
292 * permissions then act as though the remote end sent
293 * us the file permissions we already have */
294 file->mode = (file->mode & ~CHMOD_BITS)
295 | (st.st_mode & CHMOD_BITS);
298 if (S_ISDIR(file->mode)) {
299 /* The file to be received is a directory, so we need
300 * to prepare appropriately. If there is already a
301 * file of that name and it is *not* a directory, then
302 * we need to delete it. If it doesn't exist, then
303 * recursively create it. */
306 return; /* TODO: causes inaccuracies -- fix */
307 if (statret == 0 && !S_ISDIR(st.st_mode)) {
308 if (robust_unlink(fname) != 0) {
309 rsyserr(FERROR, errno,
310 "recv_generator: unlink %s to make room for directory",
316 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
317 if (!(relative_paths && errno == ENOENT
318 && create_directory_path(fname, orig_umask) == 0
319 && do_mkdir(fname, file->mode) == 0)) {
320 rsyserr(FERROR, errno,
321 "recv_generator: mkdir %s failed",
325 /* f_out is set to -1 when doing final directory-permission
326 * and modification-time repair. */
327 if (set_perms(fname, file, statret ? NULL : &st, 0)
328 && verbose && f_out != -1)
329 rprintf(FINFO, "%s/\n", safe_fname(fname));
333 if (preserve_links && S_ISLNK(file->mode)) {
335 char lnk[MAXPATHLEN];
338 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
340 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
341 full_fname(fname), file->u.link);
346 l = readlink(fname,lnk,MAXPATHLEN-1);
349 /* A link already pointing to the
350 * right place -- no further action
352 if (strcmp(lnk,file->u.link) == 0) {
353 set_perms(fname, file, &st,
358 /* Not a symlink, so delete whatever's
359 * already there and put a new symlink
363 if (do_symlink(file->u.link,fname) != 0) {
364 rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
365 full_fname(fname), safe_fname(file->u.link));
367 set_perms(fname,file,NULL,0);
369 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
370 safe_fname(file->u.link));
378 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
380 st.st_mode != file->mode ||
381 st.st_rdev != file->u.rdev) {
384 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
386 (int)file->mode, (int)file->u.rdev);
388 if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
389 rsyserr(FERROR, errno, "mknod %s failed",
392 set_perms(fname,file,NULL,0);
394 rprintf(FINFO, "%s\n",
399 set_perms(fname, file, &st, PERMS_REPORT);
405 if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
408 if (!S_ISREG(file->mode)) {
409 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
416 if (statret == -1 && compare_dest != NULL) {
417 /* try the file at compare_dest instead */
418 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
419 if (link_stat(fnamecmpbuf, &st, 0) == 0
420 && S_ISREG(st.st_mode)) {
422 if (link_dest && !dry_run) {
423 if (do_link(fnamecmpbuf, fname) < 0) {
425 rsyserr(FINFO, errno,
430 fnamecmp = fnamecmpbuf;
434 fnamecmp = fnamecmpbuf;
439 if (statret == 0 && !S_ISREG(st.st_mode)) {
440 if (delete_file(fname) != 0)
447 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
449 if (stat_errno == ENOENT) {
451 if (!dry_run && !read_batch)
452 write_sum_head(f_out, NULL);
453 } else if (verbose > 1) {
454 rsyserr(FERROR, stat_errno,
455 "recv_generator: failed to stat %s",
461 if (opt_ignore_existing && fnamecmp == fname) {
463 rprintf(FINFO, "%s exists\n", safe_fname(fname));
467 if (update_only && fnamecmp == fname
468 && cmp_modtime(st.st_mtime, file->modtime) > 0) {
470 rprintf(FINFO, "%s is newer\n", safe_fname(fname));
474 if (skip_file(fnamecmp, file, &st)) {
475 if (fnamecmp == fname)
476 set_perms(fname, file, &st, PERMS_REPORT);
480 if (dry_run || read_batch) {
485 if (whole_file > 0) {
487 write_sum_head(f_out, NULL);
493 char *partialptr = partial_dir_fname(fname);
494 if (partialptr && link_stat(partialptr, &st2, 0) == 0
495 && S_ISREG(st2.st_mode)) {
497 fnamecmp = partialptr;
502 fd = do_open(fnamecmp, O_RDONLY, 0);
505 rsyserr(FERROR, errno, "failed to open %s, continuing",
506 full_fname(fnamecmp));
507 /* pretend the file didn't exist */
508 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
511 write_sum_head(f_out, NULL);
516 rprintf(FINFO, "gen mapped %s of size %.0f\n",
517 safe_fname(fnamecmp), (double)st.st_size);
521 rprintf(FINFO, "generating and sending sums for %d\n", i);
524 generate_and_send_sums(fd, st.st_size, f_out);
530 void generate_files(int f_out, struct file_list *flist, char *local_name)
534 char fbuf[MAXPATHLEN];
537 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
538 (long)getpid(), flist->count);
544 ? "delta-transmission disabled for local transfer or --whole-file\n"
545 : "delta transmission enabled\n");
548 /* we expect to just sit around now, so don't exit on a
549 timeout. If we really get a timeout then the other process should
553 for (i = 0; i < flist->count; i++) {
554 struct file_struct *file = flist->files[i];
555 struct file_struct copy;
559 /* we need to ensure that any directories we create have writeable
560 permissions initially so that we can create the files within
561 them. This is then fixed after the files are transferred */
562 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
564 /* XXX: Could this be causing a problem on SCO? Perhaps their
565 * handling of permissions is strange? */
566 copy.mode |= S_IWUSR; /* user write */
570 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
575 csum_length = SUM_LENGTH;
579 rprintf(FINFO,"generate_files phase=%d\n",phase);
581 write_int(f_out, -1);
583 /* files can cycle through the system more than once
584 * to catch initial checksum errors */
585 while ((i = get_redo_num()) != -1) {
586 struct file_struct *file = flist->files[i];
587 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
593 rprintf(FINFO,"generate_files phase=%d\n",phase);
595 write_int(f_out, -1);
597 if (preserve_hard_links)
600 /* now we need to fix any directory permissions that were
601 * modified during the transfer */
602 for (i = 0; i < flist->count; i++) {
603 struct file_struct *file = flist->files[i];
604 if (!file->basename || !S_ISDIR(file->mode))
606 recv_generator(local_name ? local_name : f_name(file),
611 rprintf(FINFO,"generate_files finished\n");