X-Git-Url: https://mattmccutchen.net/rsync/rsync-patches.git/blobdiff_plain/ae10e51ed8a104901fd8bc2611483e56c51454e9..5214a41bbae94607b196b199b483710e1babf292:/checksum-updating.diff diff --git a/checksum-updating.diff b/checksum-updating.diff index f535db4..a68de7b 100644 --- a/checksum-updating.diff +++ b/checksum-updating.diff @@ -1,609 +1,650 @@ -This adds a sender optimization feature that allows a cache of checksums -to be created/updated and used when the client specifies the --checksum -option. +This builds on the checksum-reading patch and adds the ability to +create and/or update the .rsyncsums files using extended mode args to +the --sumfiles=MODE option and the "checksum files = MODE" daemon +parameter. + +CAUTION: This patch is only lightly tested. If you're interested +in using it, please help out. To use this patch, run these commands for a successful build: + patch -p1 to_redo) ++ + static struct csum_cache { + struct file_list *flist; ++ const char *dirname; ++ int checksum_matches; ++ int checksum_updates; + } *csum_cache = NULL; + + static void flist_sort_and_clean(struct file_list *flist, int flags); +@@ -364,7 +373,79 @@ static void flist_done_allocating(struct file_list *flist) flist->pool_boundary = ptr; } -+/* The len count is the length of the basename + 1 for the null. */ -+static void add_checksum(const char *dirname, const char *basename, int len, -+ OFF_T file_length, time_t mtime, const char *sum, -+ int flags) -+{ -+ struct file_struct *file; -+ int alloc_len, extra_len; -+ char *bp; -+ -+ if (len == 8+1 && *basename == '.' -+ && (strcmp(basename, ".md5sums") == 0 -+ || strcmp(basename, ".md4sums") == 0)) -+ return; -+ -+ if (len < 0) -+ len = strlen(basename) + 1; -+ -+ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT) -+ * EXTRA_LEN; -+#if EXTRA_ROUNDING > 0 -+ if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) -+ extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; -+#endif -+ alloc_len = FILE_STRUCT_LEN + extra_len + len; -+ bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); -+ -+ memset(bp, 0, extra_len + FILE_STRUCT_LEN); -+ bp += extra_len; -+ file = (struct file_struct *)bp; -+ bp += FILE_STRUCT_LEN; -+ -+ memcpy(bp, basename, len); -+ -+ file->flags = flags; -+ file->mode = S_IFREG; -+ file->modtime = mtime; -+ file->len32 = (uint32)file_length; -+ if (file_length > 0xFFFFFFFFu) { -+ file->flags |= FLAG_LENGTH64; -+ OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); -+ } -+ file->dirname = dirname; -+ bp = (char*)F_SUM(file); -+ memcpy(bp, sum, checksum_len); -+ -+ flist_expand(checksum_flist, 1); -+ checksum_flist->files[checksum_flist->count++] = file; -+ -+ checksum_flist->sorted = checksum_flist->files; -+} -+ -+/* The direname value must remain unchanged during the lifespan of the -+ * created checksum_flist object because we use it directly. */ -+static void read_checksums(const char *dirname) +-void reset_checksum_cache() ++static void checksum_filename(int slot, const char *dirname, char *fbuf) +{ -+ char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; -+ const char *filename; -+ OFF_T file_length; -+ time_t mtime; -+ int len, dlen, i, flags; -+ char *cp; -+ FILE *fp; -+ -+ if (checksum_flist) { -+ /* Reset the pool memory and empty the file-list array. */ -+ pool_free_old(checksum_flist->file_pool, -+ pool_boundary(checksum_flist->file_pool, 0)); -+ checksum_flist->count = 0; -+ } else -+ checksum_flist = flist_new(FLIST_TEMP, "read_checksums"); -+ -+ checksum_flist->low = 0; -+ checksum_flist->high = -1; -+ checksum_matches = 0; -+ -+ if (protocol_version >= 30) -+ filename = ".md5sums"; -+ else -+ filename = ".md4sums"; -+ if (dirname) { -+ dlen = strlcpy(fbuf, dirname, sizeof fbuf); -+ if (dlen >= (int)sizeof fbuf) ++ if (dirname && *dirname) { ++ unsigned int len; ++ if (slot) { ++ len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN); ++ if (len >= MAXPATHLEN) ++ return; ++ } else ++ len = 0; ++ if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len) + return; -+ fbuf[dlen++] = '/'; + } else -+ dlen = 0; -+ strlcpy(fbuf+dlen, filename, sizeof fbuf - dlen); -+ if (!(fp = fopen(fbuf, "r"))) -+ return; -+ -+ while (fgets(line, sizeof line, fp)) { -+ for (i = 0, cp = line; i < checksum_len*2; i++, cp++) { -+ int x; -+ if (isDigit(cp)) -+ x = *cp - '0'; -+ else if (isAlpha(cp)) { -+ x = (*cp & 0xF) + 9; -+ if (x > 0xF) { -+ cp = ""; -+ break; -+ } -+ } else { -+ cp = ""; -+ break; -+ } -+ if (i & 1) -+ sum[i/2] |= x; -+ else -+ sum[i/2] = x << 4; -+ } -+ -+ if (*cp != ' ') -+ continue; -+ while (*++cp == ' ') {} -+ -+ file_length = 0; -+ while (isDigit(cp)) -+ file_length = file_length * 10 + *cp++ - '0'; -+ -+ if (*cp != ' ') -+ continue; -+ while (*++cp == ' ') {} -+ -+ mtime = 0; -+ while (isDigit(cp)) -+ mtime = mtime * 10 + *cp++ - '0'; -+ -+ if (*cp != ' ') -+ continue; -+ while (*++cp == ' ') {} -+ -+ len = strlen(cp); -+ while (len && (cp[len-1] == '\n' || cp[len-1] == '\r')) -+ len--; -+ if (!len) -+ continue; -+ cp[len++] = '\0'; /* len now counts the null */ -+ if (strchr(cp, '/') || len > MAXPATHLEN) -+ continue; -+ -+ strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); -+ if (is_excluded(fbuf, 0, ALL_FILTERS)) { -+ flags = FLAG_FILE_SENT; -+ checksum_matches++; -+ } else -+ flags = 0; -+ -+ add_checksum(dirname, cp, len, file_length, mtime, sum, flags); -+ } -+ fclose(fp); -+ -+ clean_flist(checksum_flist, 0); ++ strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN); +} + -+static void write_checksums(const char *dirname) ++static void write_checksums(int slot, struct file_list *flist, int whole_dir) +{ -+ char buf[MAXPATHLEN+1024]; -+ const char *filename; -+ int new_entries = checksum_flist->count > checksum_flist->high + 1; -+ int orphan_entires = checksum_flist->count != checksum_matches; -+ FILE *out_fp; + int i; ++ FILE *out_fp; ++ char fbuf[MAXPATHLEN]; ++ int new_entries = csum_cache[slot].checksum_updates != 0; ++ int counts_match = flist->used == csum_cache[slot].checksum_matches; ++ int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0; ++ const char *dirname = csum_cache[slot].dirname; + -+ if (dry_run) -+ return; -+ -+ for (i = checksum_flist->high + 1; i < checksum_flist->count; i++) { -+ struct file_struct *file = checksum_flist->sorted[i]; -+ file->flags |= FLAG_FILE_SENT; -+ } ++ flist_sort_and_clean(flist, 0); + -+ clean_flist(checksum_flist, 0); ++ if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) ++ return; + -+ if (protocol_version >= 30) -+ filename = ".md5sums"; -+ else -+ filename = ".md4sums"; -+ if (dirname) { -+ if (pathjoin(buf, sizeof buf, dirname, filename) >= sizeof buf) -+ return; -+ } else -+ strlcpy(buf, filename, sizeof buf); ++ checksum_filename(slot, dirname, fbuf); + -+ if (checksum_flist->high - checksum_flist->low < 0) { -+ unlink(buf); ++ if (flist->high - flist->low < 0 && no_skipped) { ++ unlink(fbuf); + return; + } + -+ if (!new_entries && !orphan_entires) ++ if (!new_entries && (counts_match || !whole_dir)) + return; + -+ if (!(out_fp = fopen(buf, "w"))) ++ if (!(out_fp = fopen(fbuf, "w"))) + return; + -+ for (i = checksum_flist->low; i <= checksum_flist->high; i++) { -+ struct file_struct *file = checksum_flist->sorted[i]; ++ for (i = flist->low; i <= flist->high; i++) { ++ struct file_struct *file = flist->sorted[i]; + const char *cp = F_SUM(file); + const char *end = cp + checksum_len; -+ if (!(file->flags & FLAG_FILE_SENT)) ++ const char *alt_sum = file->basename + strlen(file->basename) + 1; ++ if (whole_dir && !(file->flags & FLAG_SUM_KEEP)) + continue; -+ while (cp != end) -+ fprintf(out_fp, "%02x", CVAL(cp++, 0)); -+ fprintf(out_fp, " %10.0f %10ld %s\n", -+ (double)F_LENGTH(file), (long)file->modtime, -+ file->basename); ++ if (protocol_version >= 30) ++ fprintf(out_fp, "%s ", alt_sum); ++ if (file->flags & FLAG_SUM_MISSING) { ++ do { ++ fputs("==", out_fp); ++ } while (++cp != end); ++ } else { ++ do { ++ fprintf(out_fp, "%02x", (int)CVAL(cp, 0)); ++ } while (++cp != end); ++ } ++ if (protocol_version < 30) ++ fprintf(out_fp, " %s", alt_sum); ++ fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", ++ (double)F_LENGTH(file), (double)file->modtime, ++ (long)F_CTIME(file), (long)F_INODE(file), file->basename); + } + + fclose(out_fp); +} + - int push_pathname(const char *dir, int len) ++void reset_checksum_cache(int whole_dir) { - if (dir == pathname) -@@ -973,34 +1187,24 @@ static struct file_struct *recv_file_ent - return file; + int slot, slots = am_sender ? 1 : basis_dir_cnt + 1; + +@@ -378,6 +459,9 @@ void reset_checksum_cache() + struct file_list *flist = csum_cache[slot].flist; + + if (flist) { ++ if (checksum_files & CSF_UPDATE && flist->next) ++ write_checksums(slot, flist, whole_dir); ++ + /* Reset the pool memory and empty the file-list array. */ + pool_free_old(flist->file_pool, + pool_boundary(flist->file_pool, 0)); +@@ -388,6 +472,10 @@ void reset_checksum_cache() + flist->low = 0; + flist->high = -1; + flist->next = NULL; ++ ++ csum_cache[slot].checksum_matches = 0; ++ csum_cache[slot].checksum_updates = 0; ++ REGULAR_SKIPPED(flist) = 0; + } } --/** -- * Create a file_struct for a named file by reading its stat() -- * information and performing extensive checks against global -- * options. -- * -- * @return the new file, or NULL if there was an error or this file -- * should be excluded. -+/* Create a file_struct for a named file by reading its stat() information -+ * and performing extensive checks against global options. - * -- * @todo There is a small optimization opportunity here to avoid -- * stat()ing the file in some circumstances, which has a certain cost. -- * We are called immediately after doing readdir(), and so we may -- * already know the d_type of the file. We could for example avoid -- * statting directories if we're not recursing, but this is not a very -- * important case. Some systems may not have d_type. -- **/ -+ * Returns a pointer to the new file struct, or NULL if there was an error -+ * or this file should be excluded. */ - struct file_struct *make_file(const char *fname, struct file_list *flist, - STRUCT_STAT *stp, int flags, int filter_level) +@@ -395,7 +483,7 @@ void reset_checksum_cache() + static int add_checksum(struct file_list *flist, const char *dirname, + const char *basename, int basename_len, OFF_T file_length, + time_t mtime, uint32 ctime, uint32 inode, +- const char *sum) ++ const char *sum, const char *alt_sum, int flags) { - static char *lastdir; -- static int lastdir_len = -1; -+ static int lastdir_len = -2; struct file_struct *file; -- STRUCT_STAT st; - char thisname[MAXPATHLEN]; - char linkname[MAXPATHLEN]; - int alloc_len, basename_len, linkname_len; - int extra_len = file_extra_cnt * EXTRA_LEN; - const char *basename; - alloc_pool_t *pool; -+ STRUCT_STAT st; - char *bp; + int alloc_len, extra_len; +@@ -412,7 +500,7 @@ static int add_checksum(struct file_list *flist, const char *dirname, + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; + #endif +- alloc_len = FILE_STRUCT_LEN + extra_len + basename_len; ++ alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + checksum_len*2 + 1; + bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum"); - if (strlcpy(thisname, fname, sizeof thisname) -@@ -1115,9 +1319,16 @@ struct file_struct *make_file(const char - memcpy(lastdir, thisname, len); - lastdir[len] = '\0'; - lastdir_len = len; -+ if (always_checksum && am_sender && flist) -+ read_checksums(lastdir); + memset(bp, 0, extra_len + FILE_STRUCT_LEN); +@@ -421,7 +509,14 @@ static int add_checksum(struct file_list *flist, const char *dirname, + bp += FILE_STRUCT_LEN; + + memcpy(bp, basename, basename_len); ++ if (alt_sum) ++ strlcpy(bp+basename_len, alt_sum, checksum_len*2 + 1); ++ else { ++ memset(bp+basename_len, '=', checksum_len*2); ++ bp[basename_len+checksum_len*2] = '\0'; ++ } + ++ file->flags = flags; + file->mode = S_IFREG; + file->modtime = mtime; + file->len32 = (uint32)file_length; +@@ -450,10 +545,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; + FILE *fp; + char *cp; +- int len, i; + time_t mtime; ++ int len, i, flags; + OFF_T file_length; + uint32 ctime, inode; ++ const char *alt_sum = NULL; + int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0; + + if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN)) +@@ -474,7 +570,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam + while (fgets(line, sizeof line, fp)) { + cp = line; + if (protocol_version >= 30) { +- char *alt_sum = cp; ++ alt_sum = cp; + if (*cp == '=') + while (*++cp == '=') {} + else +@@ -485,7 +581,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam } -- } else -+ } else { - basename = thisname; -+ if (always_checksum && am_sender && flist && lastdir_len == -2) { -+ lastdir_len = -1; -+ read_checksums(NULL); + + if (*cp == '=') { +- continue; ++ for (i = 0; i < checksum_len*2; i++, cp++) { ++ if (*cp != '=') { ++ cp = ""; ++ break; ++ } ++ } ++ memset(sum, 0, checksum_len); ++ flags = FLAG_SUM_MISSING; + } else { + for (i = 0; i < checksum_len*2; i++, cp++) { + int x; +@@ -503,13 +606,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam + else + sum[i/2] = x << 4; + } ++ flags = 0; + } + if (*cp != ' ') + break; + while (*++cp == ' ') {} + + if (protocol_version < 30) { +- char *alt_sum = cp; ++ alt_sum = cp; + if (*cp == '=') + while (*++cp == '=') {} + else +@@ -559,24 +663,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam + continue; + + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); ++ if (is_excluded(fbuf, 0, ALL_FILTERS)) { ++ flags |= FLAG_SUM_KEEP; ++ csum_cache[slot].checksum_matches++; + } -+ } - basename_len = strlen(basename) + 1; /* count the '\0' */ - #ifdef SUPPORT_LINKS -@@ -1193,11 +1404,30 @@ struct file_struct *make_file(const char + add_checksum(flist, dirname, cp, len, file_length, + mtime, ctime, inode, +- sum); ++ sum, alt_sum, flags); } - #endif + fclose(fp); -- if (always_checksum && am_sender && S_ISREG(st.st_mode)) -- file_checksum(thisname, tmp_sum, st.st_size); -- - F_PATHNAME(file) = pathname; - -+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) { -+ int j; -+ if (flist && (j = flist_find(checksum_flist, file)) >= 0) { -+ struct file_struct *fp = checksum_flist->sorted[j]; -+ if (fp->modtime == file->modtime && F_LENGTH(fp) == F_LENGTH(file)) { -+ memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); -+ fp->flags |= FLAG_FILE_SENT; -+ checksum_matches++; -+ } else { -+ clear_file(fp); -+ goto compute_checksum; -+ } -+ } else { -+ compute_checksum: -+ file_checksum(thisname, tmp_sum, st.st_size); -+ if (checksum_updating && flist) { -+ add_checksum(file->dirname, basename, basename_len, -+ st.st_size, st.st_mtime, tmp_sum, 0); -+ } + flist_sort_and_clean(flist, CLEAN_KEEP_LAST); + } + ++void set_cached_checksum(struct file_list *file_flist, struct file_struct *file) ++{ ++ int j; ++ FILE *out_fp; ++ STRUCT_STAT st; ++ char fbuf[MAXPATHLEN]; ++ const char *fn = f_name(file, NULL); ++ struct file_list *flist = csum_cache[0].flist; ++ ++ if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) ++ return; ++ ++ if (stat(fn, &st) < 0) ++ return; ++ ++ checksum_filename(0, file->dirname, fbuf); ++ ++ if (file_flist != flist->next) { ++ const char *cp = F_SUM(file); ++ const char *end = cp + checksum_len; ++ ++ if (!(out_fp = fopen(fbuf, "a"))) ++ return; ++ ++ if (protocol_version >= 30) { ++ for (j = 0; j < checksum_len; j++) ++ fputs("==", out_fp); ++ fputc(' ', out_fp); + } ++ do { ++ fprintf(out_fp, "%02x", (int)CVAL(cp, 0)); ++ } while (++cp != end); ++ if (protocol_version < 30) { ++ fputc(' ', out_fp); ++ for (j = 0; j < checksum_len; j++) ++ fputs("==", out_fp); ++ } ++ fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", ++ (double)st.st_size, (double)st.st_mtime, ++ (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino, ++ file->basename); ++ ++ fclose(out_fp); ++ return; ++ } ++ ++ if ((j = flist_find(flist, file)) >= 0) { ++ struct file_struct *fp = flist->sorted[j]; ++ int inc = 0; ++ if (F_LENGTH(fp) != st.st_size) { ++ fp->len32 = (uint32)st.st_size; ++ if (st.st_size > 0xFFFFFFFFu) { ++ OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32); ++ fp->flags |= FLAG_LENGTH64; ++ } else ++ fp->flags &= FLAG_LENGTH64; ++ inc = 1; ++ } ++ if (fp->modtime != st.st_mtime) { ++ fp->modtime = st.st_mtime; ++ inc = 1; ++ } ++ if (F_CTIME(fp) != (uint32)st.st_ctime) { ++ F_CTIME(fp) = (uint32)st.st_ctime; ++ inc = 1; ++ } ++ if (F_INODE(fp) != (uint32)st.st_ino) { ++ F_INODE(fp) = (uint32)st.st_ino; ++ inc = 1; ++ } ++ memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN); ++ csum_cache[0].checksum_updates += inc; ++ fp->flags &= ~FLAG_SUM_MISSING; ++ fp->flags |= FLAG_SUM_KEEP; ++ return; + } + - /* This code is only used by the receiver when it is building - * a list of files for a delete pass. */ - if (keep_dirlinks && linkname_len && flist) { -@@ -1241,14 +1471,14 @@ void unmake_file(struct file_struct *fil - - static struct file_struct *send_file_name(int f, struct file_list *flist, - char *fname, STRUCT_STAT *stp, -- int flags, int filter_flags) -+ int flags, int filter_level) ++ csum_cache[0].checksum_updates += ++ add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1, ++ st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime, ++ st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP); ++} ++ + void get_cached_checksum(int slot, const char *fname, struct file_struct *file, +- STRUCT_STAT *stp, char *sum_buf) ++ int basename_len, STRUCT_STAT *stp, char *sum_buf) { - struct file_struct *file; - #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS - statx sx; - #endif + struct file_list *flist = csum_cache[slot].flist; + int j; -- file = make_file(fname, flist, stp, flags, filter_flags); -+ file = make_file(fname, flist, stp, flags, filter_level); - if (!file) - return NULL; + if (!flist->next) { + flist->next = cur_flist; /* next points from checksum flist to file flist */ ++ csum_cache[slot].dirname = file->dirname; + read_checksums(slot, flist, file->dirname); + } -@@ -1442,7 +1672,7 @@ static void send_directory(int f, struct - DIR *d; - int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0; - int start = flist->count; -- int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS; -+ int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; +@@ -588,12 +780,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file, + && (checksum_files & CSF_LAX + || (F_CTIME(fp) == (uint32)stp->st_ctime + && F_INODE(fp) == (uint32)stp->st_ino))) { +- memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); ++ if (fp->flags & FLAG_SUM_MISSING) { ++ fp->flags &= ~FLAG_SUM_MISSING; ++ csum_cache[slot].checksum_updates++; ++ file_checksum(fname, stp->st_size, sum_buf); ++ memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN); ++ } else { ++ csum_cache[slot].checksum_matches++; ++ memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); ++ } ++ fp->flags |= FLAG_SUM_KEEP; + return; + } ++ clear_file(fp); + } - assert(flist != NULL); + file_checksum(fname, stp->st_size, sum_buf); ++ ++ if (checksum_files & CSF_UPDATE) { ++ if (basename_len < 0) ++ basename_len = strlen(file->basename) + 1; ++ csum_cache[slot].checksum_updates += ++ add_checksum(flist, file->dirname, file->basename, basename_len, ++ stp->st_size, stp->st_mtime, (uint32)stp->st_ctime, ++ (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP); ++ } + } -@@ -1471,7 +1701,7 @@ static void send_directory(int f, struct - continue; - } + /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's +@@ -1488,6 +1699,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { + if (ignore_perishable) + non_perishable_cnt++; ++ if (S_ISREG(st.st_mode)) ++ REGULAR_SKIPPED(flist)++; + return NULL; + } -- send_file_name(f, flist, fbuf, NULL, flags, filter_flags); -+ send_file_name(f, flist, fbuf, NULL, flags, filter_level); +@@ -1534,13 +1747,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + lastdir[len] = '\0'; + lastdir_len = len; + if (checksum_files && am_sender && flist) +- reset_checksum_cache(); ++ reset_checksum_cache(0); + } + } else { + basename = thisname; + if (checksum_files && am_sender && flist && lastdir_len == -2) { + lastdir_len = -1; +- reset_checksum_cache(); ++ reset_checksum_cache(0); + } } + basename_len = strlen(basename) + 1; /* count the '\0' */ +@@ -1646,7 +1859,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, - fbuf[len] = '\0'; -@@ -1483,6 +1713,9 @@ static void send_directory(int f, struct + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { + if (flist && checksum_files) +- get_cached_checksum(0, thisname, file, &st, tmp_sum); ++ get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum); + else + file_checksum(thisname, st.st_size, tmp_sum); + if (sender_keeps_checksum) +@@ -2019,6 +2232,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, closedir(d); -+ if (checksum_updating && always_checksum && am_sender && f >= 0) -+ write_checksums(fbuf); ++ if (checksum_files & CSF_UPDATE && am_sender && f >= 0) ++ reset_checksum_cache(1); + if (f >= 0 && recurse && !divert_dirs) { - int i, end = flist->count - 1; - /* send_if_directory() bumps flist->count, so use "end". */ -@@ -2206,7 +2439,7 @@ void flist_free(struct file_list *flist) - - if (!flist->prev || !flist_cnt) - pool_destroy(flist->file_pool); -- else -+ else if (flist->pool_boundary) - pool_free_old(flist->file_pool, flist->pool_boundary); - - if (flist->sorted && flist->sorted != flist->files) -@@ -2225,6 +2458,7 @@ static void clean_flist(struct file_list - if (!flist) - return; - if (flist->count == 0) { -+ flist->low = 0; - flist->high = -1; - return; + int i, end = flist->used - 1; + /* send_if_directory() bumps flist->used, so use "end". */ +@@ -2652,6 +2868,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) + rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i()); } ---- old/loadparm.c -+++ new/loadparm.c -@@ -149,6 +149,7 @@ typedef struct - int syslog_facility; - int timeout; - -+ BOOL checksum_updating; - BOOL fake_super; - BOOL ignore_errors; - BOOL ignore_nonreadable; -@@ -197,6 +198,7 @@ static service sDefault = - /* syslog_facility; */ LOG_DAEMON, - /* timeout; */ 0, - -+ /* checksum_updating; */ False, - /* fake_super; */ False, - /* ignore_errors; */ False, - /* ignore_nonreadable; */ False, -@@ -313,6 +315,7 @@ static struct parm_struct parm_table[] = - {"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0}, - {"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0}, - {"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0}, -+ {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0}, - {"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0}, - {"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0}, - {"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0}, -@@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super) - FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) - FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) - FN_LOCAL_BOOL(lp_list, list) -+FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating) - FN_LOCAL_BOOL(lp_read_only, read_only) - FN_LOCAL_BOOL(lp_strict_modes, strict_modes) - FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging) ---- old/options.c -+++ new/options.c -@@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0; - int ignore_existing = 0; - int ignore_non_existing = 0; - int need_messages_from_generator = 0; -+int checksum_updating = 0; - int max_delete = -1; - OFF_T max_size = 0; - OFF_T min_size = 0; -@@ -302,6 +303,7 @@ void usage(enum logcode F) - rprintf(F," -q, --quiet suppress non-error messages\n"); - rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); - rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); -+ rprintf(F," --checksum-updating sender updates .md[45]sums files\n"); - rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); - rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); - rprintf(F," -r, --recursive recurse into directories\n"); -@@ -542,6 +544,7 @@ static struct poptOption long_options[] - {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, - {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, - {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, -+ {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 }, - {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, - {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, - {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, -@@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg - args[ac++] = basis_dir[i]; + ++ if (checksum_files & CSF_UPDATE && flist_eof) ++ reset_checksum_cache(0); /* writes any last updates */ ++ + return flist; + } + +diff --git a/generator.c b/generator.c +--- a/generator.c ++++ b/generator.c +@@ -111,6 +111,7 @@ static int dir_tweaking; + static int symlink_timeset_failed_flags; + static int need_retouch_dir_times; + static int need_retouch_dir_perms; ++static int started_whole_dir, upcoming_whole_dir; + static const char *solo_file = NULL; + + enum nonregtype { +@@ -533,7 +534,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot + if (always_checksum > 0 && S_ISREG(st->st_mode)) { + char sum[MAX_DIGEST_LEN]; + if (checksum_files && slot >= 0) +- get_cached_checksum(slot, fn, file, st, sum); ++ get_cached_checksum(slot, fn, file, -1, st, sum); + else + file_checksum(fn, st->st_size, sum); + return memcmp(sum, F_SUM(file), checksum_len) == 0; +@@ -1184,7 +1185,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); + } + if (checksum_files) { +- reset_checksum_cache(); ++ reset_checksum_cache(started_whole_dir); ++ started_whole_dir = upcoming_whole_dir; } + need_new_dirscan = 0; } -- } -+ } else if (checksum_updating) -+ args[ac++] = "--checksum-updating"; -+ +@@ -1342,6 +1344,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + else + change_local_filter_dir(fname, strlen(fname), F_DEPTH(file)); + } ++ upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0; + goto cleanup; + } - if (append_mode) - args[ac++] = "--append"; ---- old/rsync.h -+++ new/rsync.h -@@ -1070,6 +1070,12 @@ isDigit(const char *ptr) - } +@@ -1615,6 +1618,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + handle_partial_dir(partialptr, PDIR_DELETE); + } + set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT); ++ if (checksum_files & CSF_UPDATE) ++ set_cached_checksum(cur_flist, file); + if (itemizing) + itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); + #ifdef SUPPORT_HARD_LINKS +@@ -2122,6 +2127,7 @@ void generate_files(int f_out, const char *local_name) + } else + change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp)); + } ++ upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0; + } + for (i = cur_flist->low; i <= cur_flist->high; i++) { + struct file_struct *file = cur_flist->sorted[i]; +@@ -2216,6 +2222,9 @@ void generate_files(int f_out, const char *local_name) + wait_for_receiver(); + } - static inline int -+isAlpha(const char *ptr) -+{ -+ return isalpha(*(unsigned char *)ptr); -+} ++ if (checksum_files) ++ reset_checksum_cache(started_whole_dir); + -+static inline int - isPrint(const char *ptr) - { - return isprint(*(unsigned char *)ptr); ---- old/rsync.yo -+++ new/rsync.yo -@@ -307,6 +307,7 @@ to the detailed description below for a - -q, --quiet suppress non-error messages - --no-motd suppress daemon-mode MOTD (see caveat) - -c, --checksum skip based on checksum, not mod-time & size -+ --checksum-updating sender updates .md[45]sums files - -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) - --no-OPTION turn off an implied OPTION (e.g. --no-D) - -r, --recursive recurse into directories -@@ -502,9 +503,9 @@ uses a "quick check" that (by default) c - of last modification match between the sender and receiver. This option - changes this to compare a 128-bit MD4 checksum for each file that has a - matching size. Generating the checksums means that both sides will expend --a lot of disk I/O reading all the data in the files in the transfer (and --this is prior to any reading that will be done to transfer changed files), --so this can slow things down significantly. -+a lot of disk I/O reading the data in all the files in the transfer, so -+this can slow things down significantly (and this is prior to any reading -+that will be done to transfer the files that have changed). - - The sending side generates its checksums while it is doing the file-system - scan that builds the list of the available files. The receiver generates -@@ -512,12 +513,43 @@ its checksums when it is scanning for ch - file that has the same size as the corresponding sender's file: files with - either a changed size or a changed checksum are selected for transfer. - -+Starting with version 3.0.0, the sending side will look for a checksum -+summary file and use a pre-generated checksum that it reads out of the file -+(as long as it matches the file's size and modified time). This allows a -+server to support the --checksum option to clients without having to -+recompute the checksums for each client. See the bf(--checksum-updating) -+option for a way to have rsync create/update the checksum files. -+ - Note that rsync always verifies that each em(transferred) file was - correctly reconstructed on the receiving side by checking a whole-file - checksum that is generated when as the file is transferred, but that - automatic after-the-transfer verification has nothing to do with this - option's before-the-transfer "Does this file need to be updated?" check. - -+dit(bf(--checksum-updating)) This option tells the sending side to create -+and/or update per-directory checksum files that are used by the -+bf(--checksum) option. The file that is updated is either .md5sums (for -+protocols >= 30) or .md4sums (for older protocols). If pre-transfer -+checksums are not being computed, this option has no effect. -+ -+The checksum files stores the computed checksum, last-known size, -+modification time, and name for each file in the current directory. If a -+later transfer finds that a file matches its prior size and modification -+time, the checksum is assumed to still be correct. Otherwise it is -+recomputed and udpated in the file. -+ -+To avoid transferring the system's checksum files, you can use an exclude -+(e.g. bf(--exclude=.md[45]sums)). To make this easier to type, you can use -+a popt alias. For instance, adding the following line in your ~/.popt file -+defines a bf(-cc) option that enables checksum updating and excludes the -+checksum files: -+ -+verb( rsync alias --cc --checksum-updating --exclude='.md[45]sums') -+ -+An rsync daemon does not allow the client to control this setting, so see -+the "checksum updating" daemon config option for information on how to make -+a daemon maintain these checksum files. -+ - dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick - way of saying you want recursion and want to preserve almost - everything (with -H being a notable omission). ---- old/rsyncd.conf.yo -+++ new/rsyncd.conf.yo -@@ -198,6 +198,21 @@ locking on this file to ensure that the - exceeded for the modules sharing the lock file. - The default is tt(/var/run/rsyncd.lock). - -+dit(bf(checksum updating)) This option tells rsync to update/create the -+checksum information in the per-directory checksum files when users copy -+files using the bf(--checksum) option. Any file that has changed since it -+was last checksummed (or is not mentioned) has its data updated in the -+.md4sums or .md5sums file (the file used depends on what protocol version -+is used for the transfer). -+ -+Note that this updating will occur even if the module is listed as being -+read-only. If you want to hide these files (and you will almost always -+want to do), add ".md[45]sums" to the module's exclude setting. -+ -+Note also that the client's command-line option, bf(--checksum-updating), -+has no effect on a daemon. A daemon will only update/create checksum files -+if this config option is true. -+ - dit(bf(read only)) The "read only" option determines whether clients + info_levels[INFO_FLIST] = save_info_flist; + info_levels[INFO_PROGRESS] = save_info_progress; + +diff --git a/io.c b/io.c +--- a/io.c ++++ b/io.c +@@ -52,6 +52,7 @@ extern int list_only; + extern int read_batch; + extern int protect_args; + extern int checksum_seed; ++extern int checksum_files; + extern int protocol_version; + extern int remove_source_files; + extern int preserve_hard_links; +@@ -987,6 +988,9 @@ static void got_flist_entry_status(enum festatus status, int ndx) + flist_ndx_push(&hlink_list, ndx); + flist->in_progress++; + } ++ } else if (checksum_files & CSF_UPDATE) { ++ struct file_struct *file = flist->files[ndx - flist->ndx_start]; ++ set_cached_checksum(flist, file); + } + break; + case FES_REDO: +diff --git a/loadparm.c b/loadparm.c +--- a/loadparm.c ++++ b/loadparm.c +@@ -312,6 +312,10 @@ static struct enum_list enum_csum_modes[] = { + { CSF_IGNORE_FILES, "none" }, + { CSF_LAX_MODE, "lax" }, + { CSF_STRICT_MODE, "strict" }, ++ { CSF_LAX_MODE|CSF_UPDATE, "+lax" }, ++ { CSF_STRICT_MODE|CSF_UPDATE, "+strict" }, ++ { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" }, ++ { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" }, + { -1, NULL } + }; + +diff --git a/options.c b/options.c +--- a/options.c ++++ b/options.c +@@ -1656,7 +1656,15 @@ int parse_arguments(int *argc_p, const char ***argv_p) + + case OPT_SUMFILES: + arg = poptGetOptArg(pc); +- checksum_files = 0; ++ if (*arg == '+') { ++ arg++; ++ checksum_files = CSF_UPDATE; ++ if (*arg == '+') { ++ arg++; ++ checksum_files |= CSF_AFFECT_DRYRUN; ++ } ++ } else ++ checksum_files = 0; + if (strcmp(arg, "lax") == 0) + checksum_files |= CSF_LAX_MODE; + else if (strcmp(arg, "strict") == 0) +diff --git a/receiver.c b/receiver.c +--- a/receiver.c ++++ b/receiver.c +@@ -47,6 +47,7 @@ extern int sparse_files; + extern int keep_partial; + extern int checksum_len; + extern int checksum_seed; ++extern int checksum_files; + extern int inplace; + extern int delay_updates; + extern mode_t orig_umask; +@@ -376,7 +377,7 @@ static void handle_delayed_updates(char *local_name) + "rename failed for %s (from %s)", + full_fname(fname), partialptr); + } else { +- if (remove_source_files ++ if (remove_source_files || checksum_files & CSF_UPDATE + || (preserve_hard_links && F_IS_HLINKED(file))) + send_msg_int(MSG_SUCCESS, ndx); + handle_partial_dir(partialptr, PDIR_DELETE); +@@ -829,7 +830,7 @@ int recv_files(int f_in, int f_out, char *local_name) + case 2: + break; + case 1: +- if (remove_source_files || inc_recurse ++ if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE + || (preserve_hard_links && F_IS_HLINKED(file))) + send_msg_int(MSG_SUCCESS, ndx); + break; +diff --git a/rsync.h b/rsync.h +--- a/rsync.h ++++ b/rsync.h +@@ -929,6 +929,8 @@ typedef struct { + + #define CSF_ENABLE (1<<1) + #define CSF_LAX (1<<2) ++#define CSF_UPDATE (1<<3) ++#define CSF_AFFECT_DRYRUN (1<<4) + + #define CSF_IGNORE_FILES 0 + #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX) +diff --git a/rsync.yo b/rsync.yo +--- a/rsync.yo ++++ b/rsync.yo +@@ -599,9 +599,13 @@ computed just as it would be if bf(--sumfiles) was not specified. + + The MODE value is either "lax", for relaxed checking (which compares size + and mtime), "strict" (which also compares ctime and inode), or "none" to +-ignore any .rsyncsums files ("none" is the default). Rsync does not create +-or update these files, but there is a perl script in the support directory +-named "rsyncsums" that can be used for that. ++ignore any .rsyncsums files ("none" is the default). ++If you want rsync to create and/or update these files, specify a prefixed ++plus ("+lax" or "+strict"). ++Adding a second prefixed '+' causes the checksum-file updates to happen ++even when the transfer is in bf(--dry-run) mode ("++lax" or "++strict"). ++There is also a perl script in the support directory named "rsyncsums" ++that can be used to update the .rsyncsums files. + + This option has no effect unless bf(--checksum, -c) was also specified. It + also only affects the current side of the transfer, so if you want the +diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo +--- a/rsyncd.conf.yo ++++ b/rsyncd.conf.yo +@@ -315,13 +315,15 @@ The default is tt(/var/run/rsyncd.lock). + dit(bf(checksum files)) This parameter tells rsync to make use of any cached + checksum information it finds in per-directory .rsyncsums files when the + current transfer is using the bf(--checksum) option. The value can be set +-to either "lax", "strict", or "none" -- see the client's bf(--sumfiles) +-option for what these choices do. ++to either "lax", "strict", "+lax", "+strict", "++lax", "++strict", or ++"none". See the client's bf(--sumfiles) option for what these choices do. + + Note also that the client's command-line option, bf(--sumfiles), has no + effect on a daemon. A daemon will only access checksum files if this +-config option tells it to. See also the bf(exclude) directive for a way +-to hide the .rsyncsums files from the user. ++config option tells it to. You can configure updating of the .rsyncsums ++files even if the module itself is configured to be read-only. See also ++the bf(exclude) directive for a way to hide the .rsyncsums files from the ++user. + + dit(bf(read only)) This parameter determines whether clients will be able to upload files or not. If "read only" is true then any - attempted uploads will fail. If "read only" is false then uploads will