1 This adds a sender optimization feature that allows a cache of checksums
2 to be used when the client specifies the --checksum option, and creates
3 and/or updates the .rsyncsums files when --checksum-updating is
6 To use this patch, run these commands for a successful build:
8 patch -p1 <patches/checksum-updating.diff
9 ./configure (optional if already run)
12 TODO: when sending individual files (as opposed to an entire directory),
13 we should still update the .rsyncsums file if we compute a new checksum.
14 (The file is currently only written if we send an entire dir.)
16 --- old/clientserver.c
17 +++ new/clientserver.c
18 @@ -37,6 +37,7 @@ extern int sanitize_paths;
19 extern int filesfrom_fd;
20 extern int remote_protocol;
21 extern int protocol_version;
22 +extern int checksum_updating;
23 extern int io_timeout;
25 extern int default_af_hint;
26 @@ -634,6 +635,8 @@ static int rsync_module(int f_in, int f_
27 else if (am_root < 0) /* Treat --fake-super from client as --super. */
30 + checksum_updating = lp_checksum_updating(i);
32 if (filesfrom_fd == 0)
45 @@ -57,6 +58,7 @@ extern int implied_dirs;
46 extern int file_extra_cnt;
47 extern int ignore_perishable;
48 extern int non_perishable_cnt;
49 +extern int checksum_updating;
50 extern int prune_empty_dirs;
51 extern int copy_links;
52 extern int copy_unsafe_links;
53 @@ -79,6 +81,9 @@ extern iconv_t ic_send, ic_recv;
55 #define PTR_SIZE (sizeof (struct file_struct *))
57 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
58 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
62 dev_t filesystem_dev; /* used to implement -x */
63 @@ -101,6 +106,9 @@ static char tmp_sum[MAX_DIGEST_LEN];
64 static char empty_sum[MAX_DIGEST_LEN];
65 static int flist_count_offset; /* for --delete --progress */
66 static int dir_count = 0;
67 +static struct file_list *checksum_flist = NULL;
68 +static int checksum_matches = 0;
69 +static int regular_skipped = 0;
71 static void clean_flist(struct file_list *flist, int strip_root);
72 static void output_flist(struct file_list *flist);
73 @@ -317,6 +325,283 @@ static void flist_done_allocating(struct
74 flist->pool_boundary = ptr;
77 +/* The len count is the length of the basename + 1 for the null. */
78 +static void add_checksum(const char *dirname, const char *basename, int len,
79 + OFF_T file_length, time_t mtime, time_t ctime,
80 + const char *sum, const char *alt_sum, int flags)
82 + struct file_struct *file;
83 + int alloc_len, extra_len;
86 + if (len == 10+1 && *basename == '.' && strcmp(basename, ".rsyncsums") == 0)
90 + len = strlen(basename) + 1;
92 + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu)
93 + + SUM_EXTRA_CNT + TIME_EXTRA_CNT)
95 +#if EXTRA_ROUNDING > 0
96 + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
97 + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
99 + alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1;
100 + bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum");
102 + memset(bp, 0, extra_len + FILE_STRUCT_LEN);
104 + file = (struct file_struct *)bp;
105 + bp += FILE_STRUCT_LEN;
107 + memcpy(bp, basename, len);
109 + strlcpy(bp+len, alt_sum, checksum_len*2 + 1);
111 + memset(bp+len, '=', checksum_len*2);
112 + bp[len+checksum_len*2] = '\0';
115 + file->flags = flags;
116 + file->mode = S_IFREG;
117 + file->modtime = mtime;
118 + file->len32 = (uint32)file_length;
119 + if (file_length > 0xFFFFFFFFu) {
120 + file->flags |= FLAG_LENGTH64;
121 + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
123 + file->dirname = dirname;
124 + bp = (char*)F_SUM(file);
125 + memcpy(bp, sum, checksum_len);
126 +#if SIZEOF_TIME_T == 4
127 + OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num = ctime;
129 + memcpy(bp - SIZEOF_TIME_T, &ctime, SIZEOF_TIME_T);
132 + flist_expand(checksum_flist, 1);
133 + checksum_flist->files[checksum_flist->count++] = file;
135 + checksum_flist->sorted = checksum_flist->files;
138 +/* The direname value must remain unchanged during the lifespan of the
139 + * created checksum_flist object because we use it directly. */
140 +static void read_checksums(const char *dirname)
142 + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
143 + const char *alt_sum = NULL;
145 + time_t mtime, ctime;
146 + int len, dlen, i, flags;
150 + if (checksum_flist) {
151 + /* Reset the pool memory and empty the file-list array. */
152 + pool_free_old(checksum_flist->file_pool,
153 + pool_boundary(checksum_flist->file_pool, 0));
154 + checksum_flist->count = 0;
156 + checksum_flist = flist_new(FLIST_TEMP, "read_checksums");
158 + checksum_flist->low = 0;
159 + checksum_flist->high = -1;
160 + checksum_matches = 0;
161 + regular_skipped = 0;
164 + dlen = strlcpy(fbuf, dirname, sizeof fbuf);
165 + if (dlen >= (int)sizeof fbuf)
167 + fbuf[dlen++] = '/';
170 + strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen);
171 + if (!(fp = fopen(fbuf, "r")))
174 + while (fgets(line, sizeof line, fp)) {
176 + if (protocol_version >= 30) {
179 + while (*++cp == '=') {}
181 + while (isXDigit(cp)) cp++;
182 + if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ')
184 + while (*++cp == ' ') {}
188 + for (i = 0; i < checksum_len*2; i++, cp++) {
194 + memset(sum, 0, checksum_len);
195 + flags = FLAG_SUM_MISSING;
197 + for (i = 0; i < checksum_len*2; i++, cp++) {
199 + if (isXDigit(cp)) {
203 + x = (*cp & 0xF) + 9;
217 + while (*++cp == ' ') {}
219 + if (protocol_version < 30) {
222 + while (*++cp == '=') {}
224 + while (isXDigit(cp)) cp++;
225 + if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ')
227 + while (*++cp == ' ') {}
231 + while (isDigit(cp))
232 + file_length = file_length * 10 + *cp++ - '0';
235 + while (*++cp == ' ') {}
238 + while (isDigit(cp))
239 + mtime = mtime * 10 + *cp++ - '0';
242 + while (*++cp == ' ') {}
245 + while (isDigit(cp))
246 + ctime = ctime * 10 + *cp++ - '0';
249 + while (*++cp == ' ') {}
252 + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r'))
256 + cp[len++] = '\0'; /* len now counts the null */
257 + if (strchr(cp, '/') || len > MAXPATHLEN)
260 + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
261 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
262 + flags |= FLAG_SUM_KEEP;
263 + checksum_matches++;
266 + add_checksum(dirname, cp, len, file_length, mtime, ctime,
267 + sum, alt_sum, flags);
271 + clean_flist(checksum_flist, 0);
274 +static void write_checksums(const char *dirname)
276 + char fbuf[MAXPATHLEN];
277 + int count = checksum_flist->count;
278 + int new_entries = count > checksum_flist->high + 1;
279 + int counts_match = count == checksum_matches;
280 + int no_skipped = regular_skipped == 0;
281 + time_t latest_time = 0;
285 + clean_flist(checksum_flist, 0);
287 + checksum_flist->count = 0;
288 + checksum_matches = 0;
289 + regular_skipped = 0;
295 + if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf)
298 + strlcpy(fbuf, ".rsyncsums", sizeof fbuf);
300 + if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) {
305 + if (!new_entries && counts_match)
308 + if (!(out_fp = fopen(fbuf, "w")))
311 + for (i = checksum_flist->low; i <= checksum_flist->high; i++) {
312 + struct file_struct *file = checksum_flist->sorted[i];
313 + const char *cp = F_SUM(file);
314 + const char *end = cp + checksum_len;
316 + if (!(file->flags & FLAG_SUM_KEEP))
318 +#if SIZEOF_TIME_T == 4
319 + ctime = OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num;
321 + memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T);
323 + if (protocol_version >= 30) {
324 + fprintf(out_fp, "%s ",
325 + file->basename + strlen(file->basename) + 1);
327 + if (file->flags & FLAG_SUM_MISSING) {
329 + fprintf(out_fp, "==");
330 + } while (++cp != end);
333 + fprintf(out_fp, "%02x", CVAL(cp, 0));
334 + } while (++cp != end);
336 + if (protocol_version < 30) {
337 + fprintf(out_fp, " %s",
338 + file->basename + strlen(file->basename) + 1);
340 + fprintf(out_fp, " %10.0f %10.0f %10.0f %s\n",
341 + (double)F_LENGTH(file), (double)file->modtime,
342 + (double)ctime, file->basename);
343 + if (file->modtime > ctime)
344 + ctime = file->modtime;
345 + if (ctime > latest_time)
346 + latest_time = ctime;
351 + set_modtime(fbuf, latest_time, latest_time);
354 int push_pathname(const char *dir, int len)
357 @@ -973,34 +1258,24 @@ static struct file_struct *recv_file_ent
362 - * Create a file_struct for a named file by reading its stat()
363 - * information and performing extensive checks against global
366 - * @return the new file, or NULL if there was an error or this file
367 - * should be excluded.
368 +/* Create a file_struct for a named file by reading its stat() information
369 + * and performing extensive checks against global options.
371 - * @todo There is a small optimization opportunity here to avoid
372 - * stat()ing the file in some circumstances, which has a certain cost.
373 - * We are called immediately after doing readdir(), and so we may
374 - * already know the d_type of the file. We could for example avoid
375 - * statting directories if we're not recursing, but this is not a very
376 - * important case. Some systems may not have d_type.
378 + * Returns a pointer to the new file struct, or NULL if there was an error
379 + * or this file should be excluded. */
380 struct file_struct *make_file(const char *fname, struct file_list *flist,
381 STRUCT_STAT *stp, int flags, int filter_level)
383 static char *lastdir;
384 - static int lastdir_len = -1;
385 + static int lastdir_len = -2;
386 struct file_struct *file;
388 char thisname[MAXPATHLEN];
389 char linkname[MAXPATHLEN];
390 int alloc_len, basename_len, linkname_len;
391 int extra_len = file_extra_cnt * EXTRA_LEN;
392 const char *basename;
397 if (strlcpy(thisname, fname, sizeof thisname)
398 @@ -1077,6 +1352,8 @@ struct file_struct *make_file(const char
399 if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
400 if (ignore_perishable)
401 non_perishable_cnt++;
402 + if (S_ISREG(st.st_mode))
407 @@ -1115,9 +1392,16 @@ struct file_struct *make_file(const char
408 memcpy(lastdir, thisname, len);
411 + if (always_checksum && am_sender && flist)
412 + read_checksums(lastdir);
417 + if (always_checksum && am_sender && flist && lastdir_len == -2) {
419 + read_checksums(NULL);
422 basename_len = strlen(basename) + 1; /* count the '\0' */
425 @@ -1193,11 +1477,44 @@ struct file_struct *make_file(const char
429 - if (always_checksum && am_sender && S_ISREG(st.st_mode))
430 - file_checksum(thisname, tmp_sum, st.st_size);
432 F_PATHNAME(file) = pathname;
434 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
436 + if (flist && (j = flist_find(checksum_flist, file)) >= 0) {
437 + struct file_struct *fp = checksum_flist->sorted[j];
439 +#if SIZEOF_TIME_T == 4
440 + ctime = OPT_EXTRA(fp, LEN64_BUMP(fp) + SUM_EXTRA_CNT)->num;
442 + memcpy(&ctime, F_SUM(fp) - SIZEOF_TIME_T, SIZEOF_TIME_T);
444 + if (F_LENGTH(fp) == st.st_size
445 + && fp->modtime == st.st_mtime && ctime == st.st_ctime) {
446 + if (fp->flags & FLAG_SUM_MISSING) {
447 + fp->flags &= ~FLAG_SUM_MISSING;
448 + file_checksum(thisname, tmp_sum, st.st_size);
449 + memcpy((char*)F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
451 + checksum_matches++;
452 + memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
454 + fp->flags |= FLAG_SUM_KEEP;
457 + goto compute_new_checksum;
460 + compute_new_checksum:
461 + file_checksum(thisname, tmp_sum, st.st_size);
462 + if (checksum_updating && flist) {
463 + add_checksum(file->dirname, basename, basename_len,
464 + st.st_size, st.st_mtime, st.st_ctime,
465 + tmp_sum, NULL, FLAG_SUM_KEEP);
470 /* This code is only used by the receiver when it is building
471 * a list of files for a delete pass. */
472 if (keep_dirlinks && linkname_len && flist) {
473 @@ -1241,14 +1558,14 @@ void unmake_file(struct file_struct *fil
475 static struct file_struct *send_file_name(int f, struct file_list *flist,
476 char *fname, STRUCT_STAT *stp,
477 - int flags, int filter_flags)
478 + int flags, int filter_level)
480 struct file_struct *file;
481 #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS
485 - file = make_file(fname, flist, stp, flags, filter_flags);
486 + file = make_file(fname, flist, stp, flags, filter_level);
490 @@ -1442,7 +1759,7 @@ static void send_directory(int f, struct
492 int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0;
493 int start = flist->count;
494 - int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
495 + int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
497 assert(flist != NULL);
499 @@ -1471,7 +1788,7 @@ static void send_directory(int f, struct
503 - send_file_name(f, flist, fbuf, NULL, flags, filter_flags);
504 + send_file_name(f, flist, fbuf, NULL, flags, filter_level);
508 @@ -1483,6 +1800,9 @@ static void send_directory(int f, struct
512 + if (checksum_updating && always_checksum && am_sender && f >= 0)
513 + write_checksums(fbuf);
515 if (f >= 0 && recurse && !divert_dirs) {
516 int i, end = flist->count - 1;
517 /* send_if_directory() bumps flist->count, so use "end". */
518 @@ -2206,7 +2526,7 @@ void flist_free(struct file_list *flist)
520 if (!flist->prev || !flist_cnt)
521 pool_destroy(flist->file_pool);
523 + else if (flist->pool_boundary)
524 pool_free_old(flist->file_pool, flist->pool_boundary);
526 if (flist->sorted && flist->sorted != flist->files)
527 @@ -2225,6 +2545,7 @@ static void clean_flist(struct file_list
530 if (flist->count == 0) {
537 @@ -149,6 +149,7 @@ typedef struct
541 + BOOL checksum_updating;
544 BOOL ignore_nonreadable;
545 @@ -197,6 +198,7 @@ static service sDefault =
546 /* syslog_facility; */ LOG_DAEMON,
549 + /* checksum_updating; */ False,
550 /* fake_super; */ False,
551 /* ignore_errors; */ False,
552 /* ignore_nonreadable; */ False,
553 @@ -313,6 +315,7 @@ static struct parm_struct parm_table[] =
554 {"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0},
555 {"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0},
556 {"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0},
557 + {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0},
558 {"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0},
559 {"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0},
560 {"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0},
561 @@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super)
562 FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
563 FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
564 FN_LOCAL_BOOL(lp_list, list)
565 +FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating)
566 FN_LOCAL_BOOL(lp_read_only, read_only)
567 FN_LOCAL_BOOL(lp_strict_modes, strict_modes)
568 FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging)
571 @@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0;
572 int ignore_existing = 0;
573 int ignore_non_existing = 0;
574 int need_messages_from_generator = 0;
575 +int checksum_updating = 0;
579 @@ -302,6 +303,7 @@ void usage(enum logcode F)
580 rprintf(F," -q, --quiet suppress non-error messages\n");
581 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
582 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
583 + rprintf(F," --checksum-updating sender updates .rsyncsums files\n");
584 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
585 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
586 rprintf(F," -r, --recursive recurse into directories\n");
587 @@ -542,6 +544,7 @@ static struct poptOption long_options[]
588 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
589 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
590 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
591 + {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 },
592 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
593 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
594 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
595 @@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg
596 args[ac++] = basis_dir[i];
600 + } else if (checksum_updating)
601 + args[ac++] = "--checksum-updating";
605 args[ac++] = "--append";
608 @@ -589,6 +589,7 @@ extern int preserve_xattrs;
609 #define EXTRA_LEN (sizeof (union file_extras))
610 #define PTR_EXTRA_LEN ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN)
611 #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN)
612 +#define TIME_EXTRA_CNT ((SIZEOF_TIME_T + EXTRA_LEN - 1) / EXTRA_LEN)
614 #define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx))
615 #define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump))
616 @@ -1070,6 +1071,12 @@ isDigit(const char *ptr)
620 +isXDigit(const char *ptr)
622 + return isxdigit(*(unsigned char *)ptr);
626 isPrint(const char *ptr)
628 return isprint(*(unsigned char *)ptr);
631 @@ -307,6 +307,7 @@ to the detailed description below for a
632 -q, --quiet suppress non-error messages
633 --no-motd suppress daemon-mode MOTD (see caveat)
634 -c, --checksum skip based on checksum, not mod-time & size
635 + --checksum-updating sender updates .rsyncsums files
636 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
637 --no-OPTION turn off an implied OPTION (e.g. --no-D)
638 -r, --recursive recurse into directories
639 @@ -502,9 +503,9 @@ uses a "quick check" that (by default) c
640 of last modification match between the sender and receiver. This option
641 changes this to compare a 128-bit MD4 checksum for each file that has a
642 matching size. Generating the checksums means that both sides will expend
643 -a lot of disk I/O reading all the data in the files in the transfer (and
644 -this is prior to any reading that will be done to transfer changed files),
645 -so this can slow things down significantly.
646 +a lot of disk I/O reading the data in all the files in the transfer, so
647 +this can slow things down significantly (and this is prior to any reading
648 +that will be done to transfer the files that have changed).
650 The sending side generates its checksums while it is doing the file-system
651 scan that builds the list of the available files. The receiver generates
652 @@ -512,12 +513,42 @@ its checksums when it is scanning for ch
653 file that has the same size as the corresponding sender's file: files with
654 either a changed size or a changed checksum are selected for transfer.
656 +Starting with version 3.0.0, the sending side will look for a checksum
657 +summary file and use a pre-generated checksum that it reads out of the file
658 +(as long as it matches the file's size and modified time). This allows a
659 +server to support the --checksum option to clients without having to
660 +recompute the checksums for each client. See the bf(--checksum-updating)
661 +option for a way to have rsync create/update these checksum files.
663 Note that rsync always verifies that each em(transferred) file was
664 correctly reconstructed on the receiving side by checking a whole-file
665 checksum that is generated when as the file is transferred, but that
666 automatic after-the-transfer verification has nothing to do with this
667 option's before-the-transfer "Does this file need to be updated?" check.
669 +dit(bf(--checksum-updating)) This option tells the sending side to create
670 +and/or update per-directory checksum files that are used by the
671 +bf(--checksum) option. The file that is updated is named .rsyncsums. If
672 +pre-transfer checksums are not being computed, this option has no effect.
674 +The checksum files stores the computed checksum, last-known size,
675 +modification time, and name for each file in the current directory. If a
676 +later transfer finds that a file matches its prior size and modification
677 +time, the checksum is assumed to still be correct. Otherwise it is
678 +recomputed and udpated in the file.
680 +To avoid transferring the system's checksum files, you can use an exclude
681 +(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use
682 +a popt alias. For instance, adding the following line in your ~/.popt file
683 +defines a bf(-cc) option that enables checksum updating and excludes the
686 +verb( rsync alias --cc --checksum-updating --exclude=.rsyncsums)
688 +An rsync daemon does not allow the client to control this setting, so see
689 +the "checksum updating" daemon config option for information on how to make
690 +a daemon maintain these checksum files.
692 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
693 way of saying you want recursion and want to preserve almost
694 everything (with -H being a notable omission).
695 --- old/rsyncd.conf.yo
696 +++ new/rsyncd.conf.yo
697 @@ -198,6 +198,20 @@ locking on this file to ensure that the
698 exceeded for the modules sharing the lock file.
699 The default is tt(/var/run/rsyncd.lock).
701 +dit(bf(checksum updating)) This option tells rsync to update/create the
702 +checksum information in the per-directory checksum files when users copy
703 +files using the bf(--checksum) option. Any file that has changed since it
704 +was last checksummed (or is not mentioned) has its data updated in the
707 +Note that this updating will occur even if the module is listed as being
708 +read-only. If you want to hide these files (and you will almost always
709 +want to do), add ".rsyncsums" to the module's exclude setting.
711 +Note also that the client's command-line option, bf(--checksum-updating),
712 +has no effect on a daemon. A daemon will only update/create checksum files
713 +if this config option is true.
715 dit(bf(read only)) The "read only" option determines whether clients
716 will be able to upload files or not. If "read only" is true then any
717 attempted uploads will fail. If "read only" is false then uploads will
718 --- old/support/rsyncsums
719 +++ new/support/rsyncsums
725 +use Cwd qw(abs_path cwd);
729 +our $SUMS_FILE = '.rsyncsums';
731 +our($recurse_opt, $force_reading, $help_opt);
734 +&Getopt::Long::Configure('bundling');
735 +&usage if !&GetOptions(
736 + 'recurse|r' => \$recurse_opt,
737 + 'force|f' => \$force_reading,
738 + 'verbose|v+' => \$verbosity,
739 + 'help|h' => \$help_opt,
742 +my $start_dir = cwd();
745 +@dirs = '.' unless @dirs;
752 +my $md4 = Digest::MD4->new;
753 +my $md5 = Digest::MD5->new;
756 + my $dir = shift @dirs;
758 + if (!chdir($dir)) {
759 + warn "Unable to chdir to $dir: $!\n";
762 + if (!opendir(DP, '.')) {
763 + warn "Unable to opendir $dir: $!\n";
769 + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
770 + print "$reldir ... ";
773 + my $sums_mtime = (stat($SUMS_FILE))[9];
778 + my $latest_time = 0;
779 + while (defined(my $fn = readdir(DP))) {
780 + next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn;
782 + push(@subdirs, "$dir/$fn");
787 + my($size,$mtime,$ctime) = (stat(_))[7,9,10];
789 + $cache{$fn} = [ $size, $mtime, $ctime ];
792 + $latest_time = $mtime if $mtime > $latest_time;
793 + $latest_time = $ctime if $ctime > $latest_time;
798 + unshift(@dirs, sort @subdirs) if $recurse_opt;
801 + if (defined $sums_mtime) {
802 + print "(removed $SUMS_FILE) " if $verbosity;
803 + unlink($SUMS_FILE);
805 + print "empty\n" if $verbosity;
809 + if (defined($sums_mtime) && $sums_mtime == $latest_time && !$force_reading) {
810 + print "OK\n" if $verbosity;
814 + if (open(FP, '+<', $SUMS_FILE)) {
817 + my($sum4, $sum5, $size, $mtime, $ctime, $fn) = split(' ', $_, 6);
818 + my $ref = $cache{$fn};
819 + if (defined $ref) {
820 + if ($$ref[0] == $size
821 + && $$ref[1] == $mtime && $$ref[2] == $ctime
822 + && $sum4 !~ /=/ && $sum5 !~ /=/) {
827 + $$ref[3] = $$ref[4] = undef;
830 + $cnt = -1; # Force rewrite due to removed line.
834 + open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n";
839 + print "updating\n" if $verbosity;
840 + while (my($fn, $ref) = each %cache) {
841 + next if defined $$ref[3] && defined $$ref[4];
842 + if (!open(IN, $fn)) {
843 + print STDERR "Unable to read $fn: $!\n";
844 + delete $cache{$fn};
848 + my($size,$mtime,$ctime) = (stat(IN))[7,9,10];
852 + while (sysread(IN, $_, 64*1024)) {
856 + $sum4 = $md4->hexdigest;
857 + $sum5 = $md5->hexdigest;
858 + print " $sum4 $sum5" if $verbosity > 2;
859 + print " $fn\n" if $verbosity > 1;
860 + my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10];
861 + last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
870 + $cache{$fn} = [ $size, $mtime, $ctime, $sum4, $sum5 ];
875 + foreach my $fn (sort keys %cache) {
876 + my $ref = $cache{$fn};
877 + my($size, $mtime, $ctime, $sum4, $sum5) = @$ref;
878 + printf FP '%s %s %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn;
880 + $latest_time = $mtime if $mtime > $latest_time;
881 + $latest_time = $ctime if $ctime > $latest_time;
883 + truncate(FP, tell(FP));
885 + print "OK.\n" if $verbosity;
890 + utime $latest_time, $latest_time, $SUMS_FILE;
896 +Usage: rsyncsums [OPTIONS] [DIRS]
899 + -r, --recurse Update $SUMS_FILE files in subdirectories too.
900 + -f, --force Force the reading of an $SUMS_FILE file that looks to be
901 + up-to-date. (Useful for weeding out old entries.)
902 + -v, --verbose Mention what we're doing. Repeat for more info.
903 + -h, --help Display this help message.