1 This builds on the checksum-reading patch and adds the ability to
2 create and/or update the .rsyncsums files using extended mode args to
3 the --sumfiles=MODE option and the "checksum files = MODE" daemon
6 CAUTION: This patch is only lightly tested. If you're interested
7 in using it, please help out.
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/checksum-reading.diff
12 patch -p1 <patches/checksum-updating.diff
13 ./configure (optional if already run)
18 - Fix the code that removes .rsyncsums files when a dir becomes empty.
20 based-on: patch/master/checksum-reading
21 diff --git a/flist.c b/flist.c
32 @@ -108,6 +109,9 @@ extern iconv_t ic_send, ic_recv;
34 #define PTR_SIZE (sizeof (struct file_struct *))
36 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
37 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
41 dev_t filesystem_dev; /* used to implement -x */
42 @@ -147,8 +151,13 @@ static char tmp_sum[MAX_DIGEST_LEN];
43 static char empty_sum[MAX_DIGEST_LEN];
44 static int flist_count_offset; /* for --delete --progress */
46 +#define REGULAR_SKIPPED(flist) ((flist)->to_redo)
48 static struct csum_cache {
49 struct file_list *flist;
50 + const char *dirname;
51 + int checksum_matches;
52 + int checksum_updates;
55 static void flist_sort_and_clean(struct file_list *flist, int flags);
56 @@ -366,7 +375,79 @@ static void flist_done_allocating(struct file_list *flist)
57 flist->pool_boundary = ptr;
60 -void reset_checksum_cache()
61 +static void checksum_filename(int slot, const char *dirname, char *fbuf)
63 + if (dirname && *dirname) {
66 + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN);
67 + if (len >= MAXPATHLEN)
71 + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len)
74 + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN);
77 +static void write_checksums(int slot, struct file_list *flist, int whole_dir)
81 + char fbuf[MAXPATHLEN];
82 + int new_entries = csum_cache[slot].checksum_updates != 0;
83 + int counts_match = flist->used == csum_cache[slot].checksum_matches;
84 + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0;
85 + const char *dirname = csum_cache[slot].dirname;
87 + flist_sort_and_clean(flist, 0);
89 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
92 + checksum_filename(slot, dirname, fbuf);
94 + if (flist->high - flist->low < 0 && no_skipped) {
99 + if (!new_entries && (counts_match || !whole_dir))
102 + if (!(out_fp = fopen(fbuf, "w")))
105 + for (i = flist->low; i <= flist->high; i++) {
106 + struct file_struct *file = flist->sorted[i];
107 + const char *cp = F_SUM(file);
108 + const char *end = cp + checksum_len;
109 + const char *alt_sum = file->basename + strlen(file->basename) + 1;
110 + if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
112 + if (protocol_version >= 30)
113 + fprintf(out_fp, "%s ", alt_sum);
114 + if (file->flags & FLAG_SUM_MISSING) {
116 + fputs("==", out_fp);
117 + } while (++cp != end);
120 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
121 + } while (++cp != end);
123 + if (protocol_version < 30)
124 + fprintf(out_fp, " %s", alt_sum);
125 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
126 + (double)F_LENGTH(file), (double)file->modtime,
127 + (long)F_CTIME(file), (long)F_INODE(file), file->basename);
133 +void reset_checksum_cache(int whole_dir)
135 int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
137 @@ -380,6 +461,9 @@ void reset_checksum_cache()
138 struct file_list *flist = csum_cache[slot].flist;
141 + if (checksum_files & CSF_UPDATE && flist->next)
142 + write_checksums(slot, flist, whole_dir);
144 /* Reset the pool memory and empty the file-list array. */
145 pool_free_old(flist->file_pool,
146 pool_boundary(flist->file_pool, 0));
147 @@ -390,6 +474,10 @@ void reset_checksum_cache()
152 + csum_cache[slot].checksum_matches = 0;
153 + csum_cache[slot].checksum_updates = 0;
154 + REGULAR_SKIPPED(flist) = 0;
158 @@ -397,7 +485,7 @@ void reset_checksum_cache()
159 static int add_checksum(struct file_list *flist, const char *dirname,
160 const char *basename, int basename_len, OFF_T file_length,
161 time_t mtime, uint32 ctime, uint32 inode,
163 + const char *sum, const char *alt_sum, int flags)
165 struct file_struct *file;
166 int alloc_len, extra_len;
167 @@ -414,7 +502,7 @@ static int add_checksum(struct file_list *flist, const char *dirname,
168 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
169 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
171 - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
172 + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + checksum_len*2 + 1;
173 bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
175 memset(bp, 0, extra_len + FILE_STRUCT_LEN);
176 @@ -423,7 +511,14 @@ static int add_checksum(struct file_list *flist, const char *dirname,
177 bp += FILE_STRUCT_LEN;
179 memcpy(bp, basename, basename_len);
181 + strlcpy(bp+basename_len, alt_sum, checksum_len*2 + 1);
183 + memset(bp+basename_len, '=', checksum_len*2);
184 + bp[basename_len+checksum_len*2] = '\0';
187 + file->flags = flags;
188 file->mode = S_IFREG;
189 file->modtime = mtime;
190 file->len32 = (uint32)file_length;
191 @@ -452,10 +547,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
192 char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
200 + const char *alt_sum = NULL;
201 int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
203 if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
204 @@ -476,7 +572,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
205 while (fgets(line, sizeof line, fp)) {
207 if (protocol_version >= 30) {
208 - char *alt_sum = cp;
211 while (*++cp == '=') {}
213 @@ -487,7 +583,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
218 + for (i = 0; i < checksum_len*2; i++, cp++) {
224 + memset(sum, 0, checksum_len);
225 + flags = FLAG_SUM_MISSING;
227 for (i = 0; i < checksum_len*2; i++, cp++) {
229 @@ -505,13 +608,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
237 while (*++cp == ' ') {}
239 if (protocol_version < 30) {
240 - char *alt_sum = cp;
243 while (*++cp == '=') {}
245 @@ -561,24 +665,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
248 strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
249 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
250 + flags |= FLAG_SUM_KEEP;
251 + csum_cache[slot].checksum_matches++;
254 add_checksum(flist, dirname, cp, len, file_length,
257 + sum, alt_sum, flags);
261 flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
264 +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file)
269 + char fbuf[MAXPATHLEN];
270 + const char *fn = f_name(file, NULL);
271 + struct file_list *flist = csum_cache[0].flist;
273 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
276 + if (stat(fn, &st) < 0)
279 + checksum_filename(0, file->dirname, fbuf);
281 + if (file_flist != flist->next) {
282 + const char *cp = F_SUM(file);
283 + const char *end = cp + checksum_len;
285 + if (!(out_fp = fopen(fbuf, "a")))
288 + if (protocol_version >= 30) {
289 + for (j = 0; j < checksum_len; j++)
290 + fputs("==", out_fp);
291 + fputc(' ', out_fp);
294 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
295 + } while (++cp != end);
296 + if (protocol_version < 30) {
297 + fputc(' ', out_fp);
298 + for (j = 0; j < checksum_len; j++)
299 + fputs("==", out_fp);
301 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
302 + (double)st.st_size, (double)st.st_mtime,
303 + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino,
310 + if ((j = flist_find(flist, file)) >= 0) {
311 + struct file_struct *fp = flist->sorted[j];
313 + if (F_LENGTH(fp) != st.st_size) {
314 + fp->len32 = (uint32)st.st_size;
315 + if (st.st_size > 0xFFFFFFFFu) {
316 + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32);
317 + fp->flags |= FLAG_LENGTH64;
319 + fp->flags &= FLAG_LENGTH64;
322 + if (fp->modtime != st.st_mtime) {
323 + fp->modtime = st.st_mtime;
326 + if (F_CTIME(fp) != (uint32)st.st_ctime) {
327 + F_CTIME(fp) = (uint32)st.st_ctime;
330 + if (F_INODE(fp) != (uint32)st.st_ino) {
331 + F_INODE(fp) = (uint32)st.st_ino;
334 + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN);
335 + csum_cache[0].checksum_updates += inc;
336 + fp->flags &= ~FLAG_SUM_MISSING;
337 + fp->flags |= FLAG_SUM_KEEP;
341 + csum_cache[0].checksum_updates +=
342 + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1,
343 + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime,
344 + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP);
347 void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
348 - STRUCT_STAT *stp, char *sum_buf)
349 + int basename_len, STRUCT_STAT *stp, char *sum_buf)
351 struct file_list *flist = csum_cache[slot].flist;
355 flist->next = cur_flist; /* next points from checksum flist to file flist */
356 + csum_cache[slot].dirname = file->dirname;
357 read_checksums(slot, flist, file->dirname);
360 @@ -590,12 +782,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
361 && (checksum_files & CSF_LAX
362 || (F_CTIME(fp) == (uint32)stp->st_ctime
363 && F_INODE(fp) == (uint32)stp->st_ino))) {
364 - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
365 + if (fp->flags & FLAG_SUM_MISSING) {
366 + fp->flags &= ~FLAG_SUM_MISSING;
367 + csum_cache[slot].checksum_updates++;
368 + file_checksum(fname, stp->st_size, sum_buf);
369 + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN);
371 + csum_cache[slot].checksum_matches++;
372 + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
374 + fp->flags |= FLAG_SUM_KEEP;
380 file_checksum(fname, stp->st_size, sum_buf);
382 + if (checksum_files & CSF_UPDATE) {
383 + if (basename_len < 0)
384 + basename_len = strlen(file->basename) + 1;
385 + csum_cache[slot].checksum_updates +=
386 + add_checksum(flist, file->dirname, file->basename, basename_len,
387 + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime,
388 + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP);
392 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
393 @@ -1490,6 +1701,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
394 if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
395 if (ignore_perishable)
396 non_perishable_cnt++;
397 + if (S_ISREG(st.st_mode))
398 + REGULAR_SKIPPED(flist)++;
402 @@ -1536,13 +1749,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
405 if (checksum_files && am_sender && flist)
406 - reset_checksum_cache();
407 + reset_checksum_cache(0);
411 if (checksum_files && am_sender && flist && lastdir_len == -2) {
413 - reset_checksum_cache();
414 + reset_checksum_cache(0);
417 basename_len = strlen(basename) + 1; /* count the '\0' */
418 @@ -1648,7 +1861,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
420 if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
421 if (flist && checksum_files)
422 - get_cached_checksum(0, thisname, file, &st, tmp_sum);
423 + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum);
425 file_checksum(thisname, st.st_size, tmp_sum);
426 if (sender_keeps_checksum)
427 @@ -2021,6 +2234,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
431 + if (checksum_files & CSF_UPDATE && am_sender && f >= 0)
432 + reset_checksum_cache(1);
434 if (f >= 0 && recurse && !divert_dirs) {
435 int i, end = flist->used - 1;
436 /* send_if_directory() bumps flist->used, so use "end". */
437 @@ -2671,6 +2887,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
438 rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
441 + if (checksum_files & CSF_UPDATE && flist_eof)
442 + reset_checksum_cache(0); /* writes any last updates */
447 diff --git a/generator.c b/generator.c
450 @@ -110,6 +110,7 @@ static int dir_tweaking;
451 static int symlink_timeset_failed_flags;
452 static int need_retouch_dir_times;
453 static int need_retouch_dir_perms;
454 +static int started_whole_dir, upcoming_whole_dir;
455 static const char *solo_file = NULL;
458 @@ -532,7 +533,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot
459 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
460 char sum[MAX_DIGEST_LEN];
461 if (checksum_files && slot >= 0)
462 - get_cached_checksum(slot, fn, file, st, sum);
463 + get_cached_checksum(slot, fn, file, -1, st, sum);
465 file_checksum(fn, st->st_size, sum);
466 return memcmp(sum, F_SUM(file), checksum_len) == 0;
467 @@ -1183,7 +1184,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
468 fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1);
470 if (checksum_files) {
471 - reset_checksum_cache();
472 + reset_checksum_cache(started_whole_dir);
473 + started_whole_dir = upcoming_whole_dir;
475 need_new_dirscan = 0;
477 @@ -1341,6 +1343,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
479 change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
481 + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0;
485 @@ -1614,6 +1617,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
486 handle_partial_dir(partialptr, PDIR_DELETE);
488 set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
489 + if (checksum_files & CSF_UPDATE)
490 + set_cached_checksum(cur_flist, file);
492 itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL);
493 #ifdef SUPPORT_HARD_LINKS
494 @@ -2115,6 +2120,7 @@ void generate_files(int f_out, const char *local_name)
496 change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
498 + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0;
500 for (i = cur_flist->low; i <= cur_flist->high; i++) {
501 struct file_struct *file = cur_flist->sorted[i];
502 @@ -2209,6 +2215,9 @@ void generate_files(int f_out, const char *local_name)
506 + if (checksum_files)
507 + reset_checksum_cache(started_whole_dir);
509 info_levels[INFO_FLIST] = save_info_flist;
510 info_levels[INFO_PROGRESS] = save_info_progress;
512 diff --git a/io.c b/io.c
515 @@ -52,6 +52,7 @@ extern int list_only;
516 extern int read_batch;
517 extern int protect_args;
518 extern int checksum_seed;
519 +extern int checksum_files;
520 extern int protocol_version;
521 extern int remove_source_files;
522 extern int preserve_hard_links;
523 @@ -1025,6 +1026,9 @@ static void got_flist_entry_status(enum festatus status, int ndx)
524 flist_ndx_push(&hlink_list, ndx);
525 flist->in_progress++;
527 + } else if (checksum_files & CSF_UPDATE) {
528 + struct file_struct *file = flist->files[ndx - flist->ndx_start];
529 + set_cached_checksum(flist, file);
533 diff --git a/loadparm.c b/loadparm.c
536 @@ -312,6 +312,10 @@ static struct enum_list enum_csum_modes[] = {
537 { CSF_IGNORE_FILES, "none" },
538 { CSF_LAX_MODE, "lax" },
539 { CSF_STRICT_MODE, "strict" },
540 + { CSF_LAX_MODE|CSF_UPDATE, "+lax" },
541 + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" },
542 + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" },
543 + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" },
547 diff --git a/options.c b/options.c
550 @@ -1657,7 +1657,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
553 arg = poptGetOptArg(pc);
554 - checksum_files = 0;
557 + checksum_files = CSF_UPDATE;
560 + checksum_files |= CSF_AFFECT_DRYRUN;
563 + checksum_files = 0;
564 if (strcmp(arg, "lax") == 0)
565 checksum_files |= CSF_LAX_MODE;
566 else if (strcmp(arg, "strict") == 0)
567 diff --git a/receiver.c b/receiver.c
570 @@ -47,6 +47,7 @@ extern int sparse_files;
571 extern int keep_partial;
572 extern int checksum_len;
573 extern int checksum_seed;
574 +extern int checksum_files;
576 extern int delay_updates;
577 extern mode_t orig_umask;
578 @@ -376,7 +377,7 @@ static void handle_delayed_updates(char *local_name)
579 "rename failed for %s (from %s)",
580 full_fname(fname), partialptr);
582 - if (remove_source_files
583 + if (remove_source_files || checksum_files & CSF_UPDATE
584 || (preserve_hard_links && F_IS_HLINKED(file)))
585 send_msg_int(MSG_SUCCESS, ndx);
586 handle_partial_dir(partialptr, PDIR_DELETE);
587 @@ -829,7 +830,7 @@ int recv_files(int f_in, int f_out, char *local_name)
591 - if (remove_source_files || inc_recurse
592 + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE
593 || (preserve_hard_links && F_IS_HLINKED(file)))
594 send_msg_int(MSG_SUCCESS, ndx);
596 diff --git a/rsync.h b/rsync.h
599 @@ -929,6 +929,8 @@ typedef struct {
601 #define CSF_ENABLE (1<<1)
602 #define CSF_LAX (1<<2)
603 +#define CSF_UPDATE (1<<3)
604 +#define CSF_AFFECT_DRYRUN (1<<4)
606 #define CSF_IGNORE_FILES 0
607 #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
608 diff --git a/rsync.yo b/rsync.yo
611 @@ -599,9 +599,13 @@ computed just as it would be if bf(--sumfiles) was not specified.
613 The MODE value is either "lax", for relaxed checking (which compares size
614 and mtime), "strict" (which also compares ctime and inode), or "none" to
615 -ignore any .rsyncsums files ("none" is the default). Rsync does not create
616 -or update these files, but there is a perl script in the support directory
617 -named "rsyncsums" that can be used for that.
618 +ignore any .rsyncsums files ("none" is the default).
619 +If you want rsync to create and/or update these files, specify a prefixed
620 +plus ("+lax" or "+strict").
621 +Adding a second prefixed '+' causes the checksum-file updates to happen
622 +even when the transfer is in bf(--dry-run) mode ("++lax" or "++strict").
623 +There is also a perl script in the support directory named "rsyncsums"
624 +that can be used to update the .rsyncsums files.
626 This option has no effect unless bf(--checksum, -c) was also specified. It
627 also only affects the current side of the transfer, so if you want the
628 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
631 @@ -315,13 +315,15 @@ The default is tt(/var/run/rsyncd.lock).
632 dit(bf(checksum files)) This parameter tells rsync to make use of any cached
633 checksum information it finds in per-directory .rsyncsums files when the
634 current transfer is using the bf(--checksum) option. The value can be set
635 -to either "lax", "strict", or "none" -- see the client's bf(--sumfiles)
636 -option for what these choices do.
637 +to either "lax", "strict", "+lax", "+strict", "++lax", "++strict", or
638 +"none". See the client's bf(--sumfiles) option for what these choices do.
640 Note also that the client's command-line option, bf(--sumfiles), has no
641 effect on a daemon. A daemon will only access checksum files if this
642 -config option tells it to. See also the bf(exclude) directive for a way
643 -to hide the .rsyncsums files from the user.
644 +config option tells it to. You can configure updating of the .rsyncsums
645 +files even if the module itself is configured to be read-only. See also
646 +the bf(exclude) directive for a way to hide the .rsyncsums files from the
649 dit(bf(read only)) This parameter determines whether clients
650 will be able to upload files or not. If "read only" is true then any