1 This builds on the checksum-reading patch and adds the ability to
2 create and/or update the .rsyncsums files using extended mode args to
3 the --sumfiles=MODE option and the "checksum files = MODE" daemon
6 CAUTION: This patch is only lightly tested. If you're interested
7 in using it, please help out.
9 To use this patch, run these commands for a successful build:
11 patch -p1 <patches/checksum-reading.diff
12 patch -p1 <patches/checksum-updating.diff
13 ./configure (optional if already run)
18 - Fix the code that removes .rsyncsums files when a dir becomes empty.
20 diff --git a/flist.c b/flist.c
31 @@ -94,6 +95,9 @@ extern iconv_t ic_send, ic_recv;
33 #define PTR_SIZE (sizeof (struct file_struct *))
35 +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
36 +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
40 dev_t filesystem_dev; /* used to implement -x */
41 @@ -133,8 +137,13 @@ static char empty_sum[MAX_DIGEST_LEN];
42 static int flist_count_offset; /* for --delete --progress */
43 static int dir_count = 0;
45 +#define REGULAR_SKIPPED(flist) ((flist)->to_redo)
47 static struct csum_cache {
48 struct file_list *flist;
49 + const char *dirname;
50 + int checksum_matches;
51 + int checksum_updates;
54 static void flist_sort_and_clean(struct file_list *flist, int flags);
55 @@ -351,7 +360,79 @@ static void flist_done_allocating(struct file_list *flist)
56 flist->pool_boundary = ptr;
59 -void reset_checksum_cache()
60 +static void checksum_filename(int slot, const char *dirname, char *fbuf)
62 + if (dirname && *dirname) {
65 + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN);
66 + if (len >= MAXPATHLEN)
70 + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len)
73 + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN);
76 +static void write_checksums(int slot, struct file_list *flist, int whole_dir)
80 + char fbuf[MAXPATHLEN];
81 + int new_entries = csum_cache[slot].checksum_updates != 0;
82 + int counts_match = flist->used == csum_cache[slot].checksum_matches;
83 + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0;
84 + const char *dirname = csum_cache[slot].dirname;
86 + flist_sort_and_clean(flist, 0);
88 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
91 + checksum_filename(slot, dirname, fbuf);
93 + if (flist->high - flist->low < 0 && no_skipped) {
98 + if (!new_entries && (counts_match || !whole_dir))
101 + if (!(out_fp = fopen(fbuf, "w")))
104 + for (i = flist->low; i <= flist->high; i++) {
105 + struct file_struct *file = flist->sorted[i];
106 + const char *cp = F_SUM(file);
107 + const char *end = cp + checksum_len;
108 + const char *alt_sum = file->basename + strlen(file->basename) + 1;
109 + if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
111 + if (protocol_version >= 30)
112 + fprintf(out_fp, "%s ", alt_sum);
113 + if (file->flags & FLAG_SUM_MISSING) {
115 + fputs("==", out_fp);
116 + } while (++cp != end);
119 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
120 + } while (++cp != end);
122 + if (protocol_version < 30)
123 + fprintf(out_fp, " %s", alt_sum);
124 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
125 + (double)F_LENGTH(file), (double)file->modtime,
126 + (long)F_CTIME(file), (long)F_INODE(file), file->basename);
132 +void reset_checksum_cache(int whole_dir)
134 int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
136 @@ -365,6 +446,9 @@ void reset_checksum_cache()
137 struct file_list *flist = csum_cache[slot].flist;
140 + if (checksum_files & CSF_UPDATE && flist->next)
141 + write_checksums(slot, flist, whole_dir);
143 /* Reset the pool memory and empty the file-list array. */
144 pool_free_old(flist->file_pool,
145 pool_boundary(flist->file_pool, 0));
146 @@ -375,6 +459,10 @@ void reset_checksum_cache()
151 + csum_cache[slot].checksum_matches = 0;
152 + csum_cache[slot].checksum_updates = 0;
153 + REGULAR_SKIPPED(flist) = 0;
157 @@ -382,7 +470,7 @@ void reset_checksum_cache()
158 static int add_checksum(struct file_list *flist, const char *dirname,
159 const char *basename, int basename_len, OFF_T file_length,
160 time_t mtime, uint32 ctime, uint32 inode,
162 + const char *sum, const char *alt_sum, int flags)
164 struct file_struct *file;
165 int alloc_len, extra_len;
166 @@ -399,7 +487,7 @@ static int add_checksum(struct file_list *flist, const char *dirname,
167 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
168 extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
170 - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
171 + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + checksum_len*2 + 1;
172 bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
174 memset(bp, 0, extra_len + FILE_STRUCT_LEN);
175 @@ -408,7 +496,14 @@ static int add_checksum(struct file_list *flist, const char *dirname,
176 bp += FILE_STRUCT_LEN;
178 memcpy(bp, basename, basename_len);
180 + strlcpy(bp+basename_len, alt_sum, checksum_len*2 + 1);
182 + memset(bp+basename_len, '=', checksum_len*2);
183 + bp[basename_len+checksum_len*2] = '\0';
186 + file->flags = flags;
187 file->mode = S_IFREG;
188 file->modtime = mtime;
189 file->len32 = (uint32)file_length;
190 @@ -437,10 +532,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
191 char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
199 + const char *alt_sum = NULL;
200 int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
202 if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
203 @@ -461,7 +557,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
204 while (fgets(line, sizeof line, fp)) {
206 if (protocol_version >= 30) {
207 - char *alt_sum = cp;
210 while (*++cp == '=') {}
212 @@ -472,7 +568,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
217 + for (i = 0; i < checksum_len*2; i++, cp++) {
223 + memset(sum, 0, checksum_len);
224 + flags = FLAG_SUM_MISSING;
226 for (i = 0; i < checksum_len*2; i++, cp++) {
228 @@ -490,13 +593,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
236 while (*++cp == ' ') {}
238 if (protocol_version < 30) {
239 - char *alt_sum = cp;
242 while (*++cp == '=') {}
244 @@ -546,24 +650,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam
247 strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
248 + if (is_excluded(fbuf, 0, ALL_FILTERS)) {
249 + flags |= FLAG_SUM_KEEP;
250 + csum_cache[slot].checksum_matches++;
253 add_checksum(flist, dirname, cp, len, file_length,
256 + sum, alt_sum, flags);
260 flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
263 +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file)
268 + char fbuf[MAXPATHLEN];
269 + const char *fn = f_name(file, NULL);
270 + struct file_list *flist = csum_cache[0].flist;
272 + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN))
275 + if (stat(fn, &st) < 0)
278 + checksum_filename(0, file->dirname, fbuf);
280 + if (file_flist != flist->next) {
281 + const char *cp = F_SUM(file);
282 + const char *end = cp + checksum_len;
284 + if (!(out_fp = fopen(fbuf, "a")))
287 + if (protocol_version >= 30) {
288 + for (j = 0; j < checksum_len; j++)
289 + fputs("==", out_fp);
290 + fputc(' ', out_fp);
293 + fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
294 + } while (++cp != end);
295 + if (protocol_version < 30) {
296 + fputc(' ', out_fp);
297 + for (j = 0; j < checksum_len; j++)
298 + fputs("==", out_fp);
300 + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
301 + (double)st.st_size, (double)st.st_mtime,
302 + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino,
309 + if ((j = flist_find(flist, file)) >= 0) {
310 + struct file_struct *fp = flist->sorted[j];
312 + if (F_LENGTH(fp) != st.st_size) {
313 + fp->len32 = (uint32)st.st_size;
314 + if (st.st_size > 0xFFFFFFFFu) {
315 + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32);
316 + fp->flags |= FLAG_LENGTH64;
318 + fp->flags &= FLAG_LENGTH64;
321 + if (fp->modtime != st.st_mtime) {
322 + fp->modtime = st.st_mtime;
325 + if (F_CTIME(fp) != (uint32)st.st_ctime) {
326 + F_CTIME(fp) = (uint32)st.st_ctime;
329 + if (F_INODE(fp) != (uint32)st.st_ino) {
330 + F_INODE(fp) = (uint32)st.st_ino;
333 + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN);
334 + csum_cache[0].checksum_updates += inc;
335 + fp->flags &= ~FLAG_SUM_MISSING;
336 + fp->flags |= FLAG_SUM_KEEP;
340 + csum_cache[0].checksum_updates +=
341 + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1,
342 + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime,
343 + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP);
346 void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
347 - STRUCT_STAT *stp, char *sum_buf)
348 + int basename_len, STRUCT_STAT *stp, char *sum_buf)
350 struct file_list *flist = csum_cache[slot].flist;
354 flist->next = cur_flist; /* next points from checksum flist to file flist */
355 + csum_cache[slot].dirname = file->dirname;
356 read_checksums(slot, flist, file->dirname);
359 @@ -575,12 +767,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
360 && (checksum_files & CSF_LAX
361 || (F_CTIME(fp) == (uint32)stp->st_ctime
362 && F_INODE(fp) == (uint32)stp->st_ino))) {
363 - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
364 + if (fp->flags & FLAG_SUM_MISSING) {
365 + fp->flags &= ~FLAG_SUM_MISSING;
366 + csum_cache[slot].checksum_updates++;
367 + file_checksum(fname, stp->st_size, sum_buf);
368 + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN);
370 + csum_cache[slot].checksum_matches++;
371 + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
373 + fp->flags |= FLAG_SUM_KEEP;
379 file_checksum(fname, stp->st_size, sum_buf);
381 + if (checksum_files & CSF_UPDATE) {
382 + if (basename_len < 0)
383 + basename_len = strlen(file->basename) + 1;
384 + csum_cache[slot].checksum_updates +=
385 + add_checksum(flist, file->dirname, file->basename, basename_len,
386 + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime,
387 + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP);
391 /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
392 @@ -1364,6 +1575,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
394 if (ignore_perishable)
395 non_perishable_cnt++;
396 + if (S_ISREG(st.st_mode))
397 + REGULAR_SKIPPED(flist)++;
401 @@ -1410,13 +1623,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
404 if (checksum_files && am_sender && flist)
405 - reset_checksum_cache();
406 + reset_checksum_cache(0);
410 if (checksum_files && am_sender && flist && lastdir_len == -2) {
412 - reset_checksum_cache();
413 + reset_checksum_cache(0);
416 basename_len = strlen(basename) + 1; /* count the '\0' */
417 @@ -1498,7 +1711,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
419 if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
420 if (flist && checksum_files)
421 - get_cached_checksum(0, thisname, file, &st, tmp_sum);
422 + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum);
424 file_checksum(thisname, st.st_size, tmp_sum);
426 @@ -1824,6 +2037,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
430 + if (checksum_files & CSF_UPDATE && am_sender && f >= 0)
431 + reset_checksum_cache(1);
433 if (f >= 0 && recurse && !divert_dirs) {
434 int i, end = flist->used - 1;
435 /* send_if_directory() bumps flist->used, so use "end". */
436 @@ -2417,6 +2633,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
441 + if (checksum_files & CSF_UPDATE && flist_eof)
442 + reset_checksum_cache(0); /* writes any last updates */
446 diff --git a/generator.c b/generator.c
449 @@ -114,6 +114,7 @@ static int dir_tweaking;
450 static int symlink_timeset_failed_flags;
451 static int need_retouch_dir_times;
452 static int need_retouch_dir_perms;
453 +static int started_whole_dir, upcoming_whole_dir;
454 static const char *solo_file = NULL;
456 /* For calling delete_item() and delete_dir_contents(). */
457 @@ -720,7 +721,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot
458 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
459 char sum[MAX_DIGEST_LEN];
460 if (checksum_files && slot >= 0)
461 - get_cached_checksum(slot, fn, file, st, sum);
462 + get_cached_checksum(slot, fn, file, -1, st, sum);
464 file_checksum(fn, st->st_size, sum);
465 return memcmp(sum, F_SUM(file), checksum_len) == 0;
466 @@ -1353,7 +1354,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
467 fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1);
469 if (checksum_files) {
470 - reset_checksum_cache();
471 + reset_checksum_cache(started_whole_dir);
472 + started_whole_dir = upcoming_whole_dir;
474 need_new_dirscan = 0;
476 @@ -1498,6 +1500,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
478 change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
480 + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0;
484 @@ -1790,6 +1793,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
485 handle_partial_dir(partialptr, PDIR_DELETE);
487 set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT);
488 + if (checksum_files & CSF_UPDATE)
489 + set_cached_checksum(cur_flist, file);
491 itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL);
492 #ifdef SUPPORT_HARD_LINKS
493 @@ -2204,6 +2209,7 @@ void generate_files(int f_out, const char *local_name)
495 change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
497 + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0;
499 for (i = cur_flist->low; i <= cur_flist->high; i++) {
500 struct file_struct *file = cur_flist->sorted[i];
501 @@ -2284,6 +2290,9 @@ void generate_files(int f_out, const char *local_name)
505 + if (checksum_files)
506 + reset_checksum_cache(started_whole_dir);
508 do_progress = save_do_progress;
509 if (delete_during == 2)
510 do_delayed_deletions(fbuf);
511 diff --git a/io.c b/io.c
514 @@ -50,6 +50,7 @@ extern int read_batch;
515 extern int csum_length;
516 extern int protect_args;
517 extern int checksum_seed;
518 +extern int checksum_files;
519 extern int protocol_version;
520 extern int remove_source_files;
521 extern int preserve_hard_links;
522 @@ -205,6 +206,9 @@ static void got_flist_entry_status(enum festatus status, const char *buf)
523 flist_ndx_push(&hlink_list, ndx);
524 flist->in_progress++;
526 + } else if (checksum_files & CSF_UPDATE) {
527 + struct file_struct *file = flist->files[ndx - flist->ndx_start];
528 + set_cached_checksum(flist, file);
532 diff --git a/loadparm.c b/loadparm.c
535 @@ -300,6 +300,10 @@ static struct enum_list enum_csum_modes[] = {
536 { CSF_IGNORE_FILES, "none" },
537 { CSF_LAX_MODE, "lax" },
538 { CSF_STRICT_MODE, "strict" },
539 + { CSF_LAX_MODE|CSF_UPDATE, "+lax" },
540 + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" },
541 + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" },
542 + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" },
546 diff --git a/options.c b/options.c
549 @@ -1233,7 +1233,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
552 arg = poptGetOptArg(pc);
553 - checksum_files = 0;
556 + checksum_files = CSF_UPDATE;
559 + checksum_files |= CSF_AFFECT_DRYRUN;
562 + checksum_files = 0;
563 if (strcmp(arg, "lax") == 0)
564 checksum_files |= CSF_LAX_MODE;
565 else if (strcmp(arg, "strict") == 0)
566 diff --git a/receiver.c b/receiver.c
569 @@ -47,6 +47,7 @@ extern int append_mode;
570 extern int sparse_files;
571 extern int keep_partial;
572 extern int checksum_seed;
573 +extern int checksum_files;
575 extern int delay_updates;
576 extern mode_t orig_umask;
577 @@ -339,7 +340,7 @@ static void handle_delayed_updates(char *local_name)
578 "rename failed for %s (from %s)",
579 full_fname(fname), partialptr);
581 - if (remove_source_files
582 + if (remove_source_files || checksum_files & CSF_UPDATE
583 || (preserve_hard_links && F_IS_HLINKED(file)))
584 send_msg_int(MSG_SUCCESS, ndx);
585 handle_partial_dir(partialptr, PDIR_DELETE);
586 @@ -716,7 +717,7 @@ int recv_files(int f_in, char *local_name)
590 - if (remove_source_files || inc_recurse
591 + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE
592 || (preserve_hard_links && F_IS_HLINKED(file)))
593 send_msg_int(MSG_SUCCESS, ndx);
595 diff --git a/rsync.h b/rsync.h
598 @@ -870,6 +870,8 @@ typedef struct {
600 #define CSF_ENABLE (1<<1)
601 #define CSF_LAX (1<<2)
602 +#define CSF_UPDATE (1<<3)
603 +#define CSF_AFFECT_DRYRUN (1<<4)
605 #define CSF_IGNORE_FILES 0
606 #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
607 diff --git a/rsync.yo b/rsync.yo
610 @@ -544,9 +544,13 @@ computed just as it would be if bf(--sumfiles) was not specified.
612 The MODE value is either "lax", for relaxed checking (which compares size
613 and mtime), "strict" (which also compares ctime and inode), or "none" to
614 -ignore any .rsyncsums files ("none" is the default). Rsync does not create
615 -or update these files, but there is a perl script in the support directory
616 -named "rsyncsums" that can be used for that.
617 +ignore any .rsyncsums files ("none" is the default).
618 +If you want rsync to create and/or update these files, specify a prefixed
619 +plus ("+lax" or "+strict").
620 +Adding a second prefixed '+' causes the checksum-file updates to happen
621 +even when the transfer is in bf(--dry-run) mode ("++lax" or "++strict").
622 +There is also a perl script in the support directory named "rsyncsums"
623 +that can be used to update the .rsyncsums files.
625 This option has no effect unless bf(--checksum, -c) was also specified. It
626 also only affects the current side of the transfer, so if you want the
627 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
630 @@ -284,13 +284,15 @@ The default is tt(/var/run/rsyncd.lock).
631 dit(bf(checksum files)) This parameter tells rsync to make use of any cached
632 checksum information it finds in per-directory .rsyncsums files when the
633 current transfer is using the bf(--checksum) option. The value can be set
634 -to either "lax", "strict", or "none" -- see the client's bf(--sumfiles)
635 -option for what these choices do.
636 +to either "lax", "strict", "+lax", "+strict", "++lax", "++strict", or
637 +"none". See the client's bf(--sumfiles) option for what these choices do.
639 Note also that the client's command-line option, bf(--sumfiles), has no
640 effect on a daemon. A daemon will only access checksum files if this
641 -config option tells it to. See also the bf(exclude) directive for a way
642 -to hide the .rsyncsums files from the user.
643 +config option tells it to. You can configure updating of the .rsyncsums
644 +files even if the module itself is configured to be read-only. See also
645 +the bf(exclude) directive for a way to hide the .rsyncsums files from the
648 dit(bf(read only)) This parameter determines whether clients
649 will be able to upload files or not. If "read only" is true then any