-This adds a sender optimization feature that allows a cache of checksums
-to be used when the client specifies the --checksum option, and creates
-and/or updates the .rsyncsums files when --checksum-updating is
-specified.
+This builds on the sender optimization feature of the checksum4mirrors
+patch and adds the ability to create and/or updates the .rsyncsums files
+when --checksum-updating (or "checksum updating = true") is specified.
To use this patch, run these commands for a successful build:
+ patch -p1 <patches/checksum4mirrors.diff
patch -p1 <patches/checksum-updating.diff
./configure (optional if already run)
make
extern int io_timeout;
extern int no_detach;
extern int default_af_hint;
-@@ -634,6 +635,8 @@ static int rsync_module(int f_in, int f_
+@@ -687,6 +688,8 @@ static int rsync_module(int f_in, int f_
else if (am_root < 0) /* Treat --fake-super from client as --super. */
am_root = 2;
--- old/flist.c
+++ new/flist.c
-@@ -25,6 +25,7 @@
+@@ -26,6 +26,7 @@
#include "io.h"
extern int verbose;
+extern int dry_run;
- extern int list_only;
extern int am_root;
extern int am_server;
-@@ -57,6 +58,7 @@ extern int implied_dirs;
+ extern int am_daemon;
+@@ -58,6 +59,7 @@ extern int implied_dirs;
extern int file_extra_cnt;
extern int ignore_perishable;
extern int non_perishable_cnt;
extern int prune_empty_dirs;
extern int copy_links;
extern int copy_unsafe_links;
-@@ -79,6 +81,9 @@ extern iconv_t ic_send, ic_recv;
+@@ -82,6 +84,9 @@ extern iconv_t ic_send, ic_recv;
#define PTR_SIZE (sizeof (struct file_struct *))
int io_error;
int checksum_len;
dev_t filesystem_dev; /* used to implement -x */
-@@ -101,6 +106,10 @@ static char tmp_sum[MAX_DIGEST_LEN];
- static char empty_sum[MAX_DIGEST_LEN];
+@@ -117,6 +122,9 @@ static char empty_sum[MAX_DIGEST_LEN];
static int flist_count_offset; /* for --delete --progress */
static int dir_count = 0;
-+static struct file_list *checksum_flist = NULL;
+ static int high_hlink_ndx;
+static int checksum_matches = 0;
+static int checksum_updates = 0;
+static int regular_skipped = 0;
+ static struct file_list *checksum_flist = NULL;
static void clean_flist(struct file_list *flist, int strip_root);
- static void output_flist(struct file_list *flist);
-@@ -317,6 +326,304 @@ static void flist_done_allocating(struct
- flist->pool_boundary = ptr;
- }
+@@ -307,7 +315,8 @@ static void flist_done_allocating(struct
-+/* The len count is the length of the basename + 1 for the null. */
-+static void add_checksum(const char *dirname, const char *basename, int len,
-+ OFF_T file_length, time_t mtime, time_t ctime,
-+ const char *sum, const char *alt_sum, int flags)
-+{
-+ struct file_struct *file;
-+ int alloc_len, extra_len;
-+ char *bp;
-+
-+ if (len == 10+1 && *basename == '.' && strcmp(basename, ".rsyncsums") == 0)
-+ return;
-+
-+ if (len < 0)
-+ len = strlen(basename) + 1;
-+
-+ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu)
-+ + SUM_EXTRA_CNT + TIME_EXTRA_CNT)
-+ * EXTRA_LEN;
-+#if EXTRA_ROUNDING > 0
-+ if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
-+ extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
-+#endif
+ /* The len count is the length of the basename + 1 for the null. */
+ static int add_checksum(const char *dirname, const char *basename, int len,
+- OFF_T file_length, time_t mtime, const char *sum)
++ OFF_T file_length, time_t mtime, int32 ctime, int32 inode,
++ const char *sum, const char *alt_sum, int flags)
+ {
+ struct file_struct *file;
+ int alloc_len, extra_len;
+@@ -318,13 +327,14 @@ static int add_checksum(const char *dirn
+ if (file_length == 0)
+ return 0;
+
+- extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT)
++ /* "2" is for a 32-bit ctime num and an 32-bit inode num. */
++ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2)
+ * EXTRA_LEN;
+ #if EXTRA_ROUNDING > 0
+ if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
+ extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
+ #endif
+- alloc_len = FILE_STRUCT_LEN + extra_len + len;
+ alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1;
-+ bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum");
-+
-+ memset(bp, 0, extra_len + FILE_STRUCT_LEN);
-+ bp += extra_len;
-+ file = (struct file_struct *)bp;
-+ bp += FILE_STRUCT_LEN;
-+
-+ memcpy(bp, basename, len);
+ bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum");
+
+ memset(bp, 0, extra_len + FILE_STRUCT_LEN);
+@@ -333,7 +343,14 @@ static int add_checksum(const char *dirn
+ bp += FILE_STRUCT_LEN;
+
+ memcpy(bp, basename, len);
+ if (alt_sum)
+ strlcpy(bp+len, alt_sum, checksum_len*2 + 1);
+ else {
+ memset(bp+len, '=', checksum_len*2);
+ bp[len+checksum_len*2] = '\0';
+ }
-+
+
+ file->flags = flags;
-+ file->mode = S_IFREG;
-+ file->modtime = mtime;
-+ file->len32 = (uint32)file_length;
-+ if (file_length > 0xFFFFFFFFu) {
-+ file->flags |= FLAG_LENGTH64;
-+ OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
-+ }
-+ file->dirname = dirname;
-+ bp = (char*)F_SUM(file);
-+ memcpy(bp, sum, checksum_len);
-+#if SIZEOF_TIME_T == 4
-+ OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num = ctime;
-+#else
-+ memcpy(bp - SIZEOF_TIME_T, &ctime, SIZEOF_TIME_T);
-+#endif
-+
-+ flist_expand(checksum_flist, 1);
-+ checksum_flist->files[checksum_flist->count++] = file;
-+
-+ checksum_flist->sorted = checksum_flist->files;
-+}
-+
+ file->mode = S_IFREG;
+ file->modtime = mtime;
+ file->len32 = (uint32)file_length;
+@@ -344,6 +361,8 @@ static int add_checksum(const char *dirn
+ file->dirname = dirname;
+ bp = F_SUM(file);
+ memcpy(bp, sum, checksum_len);
++ F_CTIME(file) = ctime;
++ F_INODE(file) = inode;
+
+ flist_expand(checksum_flist, 1);
+ checksum_flist->files[checksum_flist->used++] = file;
+@@ -353,17 +372,104 @@ static int add_checksum(const char *dirn
+ return 1;
+ }
+
+static void write_checksums(const char *next_dirname, int whole_dir)
+{
+ static const char *dirname_save;
+ char fbuf[MAXPATHLEN];
+ const char *dirname;
-+ int count, new_entries, counts_match, no_skipped;
-+ time_t latest_time = 0;
++ int used, new_entries, counts_match, no_skipped;
+ FILE *out_fp;
+ int i;
+
+ if (!dirname)
+ return;
+
-+ count = checksum_flist->count;
++ used = checksum_flist->used;
+ new_entries = checksum_updates != 0;
-+ counts_match = count == checksum_matches;
++ counts_match = used == checksum_matches;
+ no_skipped = whole_dir && regular_skipped == 0;
+
+ clean_flist(checksum_flist, 0);
+
-+ checksum_flist->count = 0;
++ checksum_flist->used = 0;
+ checksum_matches = 0;
+ checksum_updates = 0;
+ regular_skipped = 0;
+ const char *cp = F_SUM(file);
+ const char *end = cp + checksum_len;
+ const char *alt_sum = file->basename + strlen(file->basename) + 1;
-+ time_t ctime;
++ int32 ctime, inode;
+ if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
+ continue;
-+#if SIZEOF_TIME_T == 4
-+ ctime = OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num;
-+#else
-+ memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T);
-+#endif
++ ctime = F_CTIME(file);
++ inode = F_INODE(file);
+ if (protocol_version >= 30)
+ fprintf(out_fp, "%s ", alt_sum);
+ if (file->flags & FLAG_SUM_MISSING) {
+ } while (++cp != end);
+ } else {
+ do {
-+ fprintf(out_fp, "%02x", CVAL(cp, 0));
++ fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
+ } while (++cp != end);
+ }
+ if (protocol_version < 30)
+ fprintf(out_fp, " %s", alt_sum);
+ if (*alt_sum == '=')
+ new_entries++;
-+ fprintf(out_fp, " %10.0f %10.0f %10.0f %s\n",
++ fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
+ (double)F_LENGTH(file), (double)file->modtime,
-+ (double)ctime, file->basename);
-+ if (file->modtime > ctime)
-+ ctime = file->modtime;
-+ if (ctime > latest_time)
-+ latest_time = ctime;
++ (long)ctime, (long)inode, file->basename);
+ }
+
+ fclose(out_fp);
-+
-+ if (whole_dir && new_entries == 0)
-+ set_modtime(fbuf, latest_time, latest_time);
-+ else
-+ set_modtime(fbuf, latest_time-1, latest_time-1);
+}
+
-+/* The direname value must remain unchanged during the lifespan of the
-+ * created checksum_flist object because we use it directly. */
-+static void read_checksums(const char *dirname)
-+{
-+ char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
+ /* The direname value must remain unchanged during the lifespan of the
+ * created checksum_flist object because we use it directly. */
+ static void read_checksums(const char *dirname)
+ {
+ char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
+ const char *alt_sum = NULL;
-+ OFF_T file_length;
-+ time_t mtime, ctime;
+ OFF_T file_length;
+ time_t mtime;
+- int len, dlen, i;
++ int32 ctime, inode;
+ int len, dlen, i, flags;
-+ char *cp;
-+ FILE *fp;
-+
-+ write_checksums(dirname, 0);
-+
-+ if (checksum_flist) {
-+ /* Reset the pool memory and empty the file-list array. */
-+ pool_free_old(checksum_flist->file_pool,
-+ pool_boundary(checksum_flist->file_pool, 0));
-+ checksum_flist->count = 0;
-+ } else
-+ checksum_flist = flist_new(FLIST_TEMP, "read_checksums");
+ char *cp;
+ FILE *fp;
+
++ if (checksum_updating)
++ write_checksums(dirname, 0);
+
-+ checksum_flist->low = 0;
-+ checksum_flist->high = -1;
+ if (checksum_flist) {
+ /* Reset the pool memory and empty the file-list array. */
+ pool_free_old(checksum_flist->file_pool,
+@@ -374,6 +480,9 @@ static void read_checksums(const char *d
+
+ checksum_flist->low = 0;
+ checksum_flist->high = -1;
+ checksum_matches = 0;
+ checksum_updates = 0;
+ regular_skipped = 0;
-+
-+ if (!dirname)
-+ return;
-+
-+ dlen = strlcpy(fbuf, dirname, sizeof fbuf);
-+ if (dlen >= (int)sizeof fbuf)
-+ return;
-+ if (dlen)
-+ fbuf[dlen++] = '/';
-+ else
-+ dirname = NULL;
-+ strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen);
-+ if (!(fp = fopen(fbuf, "r")))
-+ return;
-+
-+ while (fgets(line, sizeof line, fp)) {
-+ cp = line;
-+ if (protocol_version >= 30) {
+
+ if (!dirname)
+ return;
+@@ -392,7 +501,7 @@ static void read_checksums(const char *d
+ while (fgets(line, sizeof line, fp)) {
+ cp = line;
+ if (protocol_version >= 30) {
+- char *alt_sum = cp;
+ alt_sum = cp;
-+ if (*cp == '=')
-+ while (*++cp == '=') {}
-+ else
-+ while (isXDigit(cp)) cp++;
-+ if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+ }
-+
-+ if (*cp == '=') {
+ if (*cp == '=')
+ while (*++cp == '=') {}
+ else
+@@ -403,7 +512,14 @@ static void read_checksums(const char *d
+ }
+
+ if (*cp == '=') {
+- continue;
+ for (i = 0; i < checksum_len*2; i++, cp++) {
+ if (*cp != '=') {
+ cp = "";
+ }
+ memset(sum, 0, checksum_len);
+ flags = FLAG_SUM_MISSING;
-+ } else {
-+ for (i = 0; i < checksum_len*2; i++, cp++) {
-+ int x;
-+ if (isXDigit(cp)) {
-+ if (isDigit(cp))
-+ x = *cp - '0';
-+ else
-+ x = (*cp & 0xF) + 9;
-+ } else {
-+ cp = "";
-+ break;
-+ }
-+ if (i & 1)
-+ sum[i/2] |= x;
-+ else
-+ sum[i/2] = x << 4;
-+ }
+ } else {
+ for (i = 0; i < checksum_len*2; i++, cp++) {
+ int x;
+@@ -421,13 +537,14 @@ static void read_checksums(const char *d
+ else
+ sum[i/2] = x << 4;
+ }
+ flags = 0;
-+ }
-+ if (*cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+
-+ if (protocol_version < 30) {
+ }
+ if (*cp != ' ')
+ break;
+ while (*++cp == ' ') {}
+
+ if (protocol_version < 30) {
+- char *alt_sum = cp;
+ alt_sum = cp;
-+ if (*cp == '=')
-+ while (*++cp == '=') {}
-+ else
-+ while (isXDigit(cp)) cp++;
-+ if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+ }
-+
-+ file_length = 0;
-+ while (isDigit(cp))
-+ file_length = file_length * 10 + *cp++ - '0';
-+ if (*cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+
-+ mtime = 0;
-+ while (isDigit(cp))
-+ mtime = mtime * 10 + *cp++ - '0';
-+ if (*cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+
+ if (*cp == '=')
+ while (*++cp == '=') {}
+ else
+@@ -451,16 +568,16 @@ static void read_checksums(const char *d
+ break;
+ while (*++cp == ' ') {}
+
+- /* Ignore ctime. */
+ ctime = 0;
-+ while (isDigit(cp))
+ while (isDigit(cp))
+- cp++;
+ ctime = ctime * 10 + *cp++ - '0';
-+ if (*cp != ' ')
-+ break;
-+ while (*++cp == ' ') {}
-+
-+ len = strlen(cp);
-+ while (len && (cp[len-1] == '\n' || cp[len-1] == '\r'))
-+ len--;
-+ if (!len)
-+ break;
-+ cp[len++] = '\0'; /* len now counts the null */
-+ if (strchr(cp, '/') || len > MAXPATHLEN)
-+ break;
-+
-+ strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
+ if (*cp != ' ')
+ break;
+ while (*++cp == ' ') {}
+
+- /* Ignore inode. */
++ inode = 0;
+ while (isDigit(cp))
+- cp++;
++ inode = inode * 10 + *cp++ - '0';
+ if (*cp != ' ')
+ break;
+ while (*++cp == ' ') {}
+@@ -477,8 +594,13 @@ static void read_checksums(const char *d
+ continue;
+
+ strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
+ if (is_excluded(fbuf, 0, ALL_FILTERS)) {
+ flags |= FLAG_SUM_KEEP;
+ checksum_matches++;
+ }
-+
-+ add_checksum(dirname, cp, len, file_length, mtime, ctime,
+
+- add_checksum(dirname, cp, len, file_length, mtime, sum);
++ add_checksum(dirname, cp, len, file_length, mtime, ctime, inode,
+ sum, alt_sum, flags);
-+ }
-+ fclose(fp);
-+
-+ clean_flist(checksum_flist, 0);
-+}
-+
- int push_pathname(const char *dir, int len)
- {
- if (dir == pathname)
-@@ -973,34 +1280,24 @@ static struct file_struct *recv_file_ent
- return file;
- }
+ }
+ fclose(fp);
--/**
-- * Create a file_struct for a named file by reading its stat()
-- * information and performing extensive checks against global
-- * options.
-- *
-- * @return the new file, or NULL if there was an error or this file
-- * should be excluded.
-+/* Create a file_struct for a named file by reading its stat() information
-+ * and performing extensive checks against global options.
- *
-- * @todo There is a small optimization opportunity here to avoid
-- * stat()ing the file in some circumstances, which has a certain cost.
-- * We are called immediately after doing readdir(), and so we may
-- * already know the d_type of the file. We could for example avoid
-- * statting directories if we're not recursing, but this is not a very
-- * important case. Some systems may not have d_type.
-- **/
-+ * Returns a pointer to the new file struct, or NULL if there was an error
-+ * or this file should be excluded. */
- struct file_struct *make_file(const char *fname, struct file_list *flist,
- STRUCT_STAT *stp, int flags, int filter_level)
- {
- static char *lastdir;
-- static int lastdir_len = -1;
-+ static int lastdir_len = -2;
- struct file_struct *file;
-- STRUCT_STAT st;
- char thisname[MAXPATHLEN];
- char linkname[MAXPATHLEN];
- int alloc_len, basename_len, linkname_len;
- int extra_len = file_extra_cnt * EXTRA_LEN;
- const char *basename;
- alloc_pool_t *pool;
-+ STRUCT_STAT st;
- char *bp;
-
- if (strlcpy(thisname, fname, sizeof thisname)
-@@ -1077,6 +1374,8 @@ struct file_struct *make_file(const char
+@@ -1260,6 +1382,8 @@ struct file_struct *make_file(const char
if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
if (ignore_perishable)
non_perishable_cnt++;
return NULL;
}
-@@ -1115,9 +1414,16 @@ struct file_struct *make_file(const char
- memcpy(lastdir, thisname, len);
- lastdir[len] = '\0';
- lastdir_len = len;
-+ if (always_checksum && am_sender && flist)
-+ read_checksums(lastdir);
- }
-- } else
-+ } else {
- basename = thisname;
-+ if (always_checksum && am_sender && flist && lastdir_len == -2) {
-+ lastdir_len = -1;
-+ read_checksums("");
-+ }
-+ }
- basename_len = strlen(basename) + 1; /* count the '\0' */
-
- #ifdef SUPPORT_LINKS
-@@ -1193,11 +1499,44 @@ struct file_struct *make_file(const char
- }
- #endif
-
-- if (always_checksum && am_sender && S_ISREG(st.st_mode))
-- file_checksum(thisname, tmp_sum, st.st_size);
--
- F_PATHNAME(file) = pathname;
-
-+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
-+ int j;
-+ if (flist && (j = flist_find(checksum_flist, file)) >= 0) {
-+ struct file_struct *fp = checksum_flist->sorted[j];
-+ time_t ctime;
-+#if SIZEOF_TIME_T == 4
-+ ctime = OPT_EXTRA(fp, LEN64_BUMP(fp) + SUM_EXTRA_CNT)->num;
-+#else
-+ memcpy(&ctime, F_SUM(fp) - SIZEOF_TIME_T, SIZEOF_TIME_T);
-+#endif
-+ if (F_LENGTH(fp) == st.st_size
-+ && fp->modtime == st.st_mtime && ctime == st.st_ctime) {
+@@ -1391,13 +1515,36 @@ struct file_struct *make_file(const char
+ int j;
+ if (flist && (j = flist_find(checksum_flist, file)) >= 0) {
+ struct file_struct *fp = checksum_flist->sorted[j];
++ int32 ctime = F_CTIME(fp);
++ int32 inode = F_INODE(fp);
+ if (F_LENGTH(fp) == st.st_size
+- && fp->modtime == st.st_mtime)
+- memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
+- else
+- file_checksum(thisname, tmp_sum, st.st_size);
+- } else
++ && fp->modtime == st.st_mtime
++ && ctime == (int32)st.st_ctime
++ && inode == (int32)st.st_ino) {
+ if (fp->flags & FLAG_SUM_MISSING) {
+ fp->flags &= ~FLAG_SUM_MISSING;
++ checksum_updates++;
+ file_checksum(thisname, tmp_sum, st.st_size);
-+ memcpy((char*)F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
++ memcpy(F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
+ } else {
+ checksum_matches++;
+ memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
+ }
+ } else {
+ compute_new_checksum:
-+ file_checksum(thisname, tmp_sum, st.st_size);
+ file_checksum(thisname, tmp_sum, st.st_size);
+ if (checksum_updating && flist) {
-+ add_checksum(file->dirname, basename, basename_len,
-+ st.st_size, st.st_mtime, st.st_ctime,
-+ tmp_sum, NULL, FLAG_SUM_KEEP);
++ checksum_updates +=
++ add_checksum(file->dirname, basename, basename_len,
++ st.st_size, st.st_mtime, st.st_ctime,
++ st.st_ino, tmp_sum, NULL, FLAG_SUM_KEEP);
+ }
+ }
-+ }
-+
- /* This code is only used by the receiver when it is building
- * a list of files for a delete pass. */
- if (keep_dirlinks && linkname_len && flist) {
-@@ -1241,14 +1580,14 @@ void unmake_file(struct file_struct *fil
-
- static struct file_struct *send_file_name(int f, struct file_list *flist,
- char *fname, STRUCT_STAT *stp,
-- int flags, int filter_flags)
-+ int flags, int filter_level)
- {
- struct file_struct *file;
- #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS
- statx sx;
- #endif
-
-- file = make_file(fname, flist, stp, flags, filter_flags);
-+ file = make_file(fname, flist, stp, flags, filter_level);
- if (!file)
- return NULL;
-
-@@ -1442,7 +1781,7 @@ static void send_directory(int f, struct
- DIR *d;
- int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0;
- int start = flist->count;
-- int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
-+ int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
-
- assert(flist != NULL);
-
-@@ -1471,7 +1810,7 @@ static void send_directory(int f, struct
- continue;
- }
-
-- send_file_name(f, flist, fbuf, NULL, flags, filter_flags);
-+ send_file_name(f, flist, fbuf, NULL, flags, filter_level);
}
- fbuf[len] = '\0';
-@@ -1483,6 +1822,9 @@ static void send_directory(int f, struct
+ /* This code is only used by the receiver when it is building
+@@ -1688,6 +1835,9 @@ static void send_directory(int f, struct
closedir(d);
+ write_checksums(NULL, 1);
+
if (f >= 0 && recurse && !divert_dirs) {
- int i, end = flist->count - 1;
- /* send_if_directory() bumps flist->count, so use "end". */
-@@ -1916,7 +2258,11 @@ struct file_list *send_file_list(int f,
- * file-list to check if this is a 1-file xfer. */
- send_extra_file_list(f, 1);
- }
-- }
-+ } else
-+ flist_eof = 1;
-+
-+ if (checksum_updating && always_checksum && flist_eof)
+ int i, end = flist->used - 1;
+ /* send_if_directory() bumps flist->used, so use "end". */
+@@ -2253,7 +2403,7 @@ struct file_list *send_file_list(int f,
+ flist_eof = 1;
+
+ if (checksum_updating && always_checksum && flist_eof)
+- read_checksums(NULL);
+ read_checksums(NULL); /* writes any last updates */
return flist;
}
-@@ -2206,7 +2552,7 @@ void flist_free(struct file_list *flist)
-
- if (!flist->prev || !flist_cnt)
- pool_destroy(flist->file_pool);
-- else
-+ else if (flist->pool_boundary)
- pool_free_old(flist->file_pool, flist->pool_boundary);
-
- if (flist->sorted && flist->sorted != flist->files)
-@@ -2225,6 +2571,7 @@ static void clean_flist(struct file_list
- if (!flist)
- return;
- if (flist->count == 0) {
-+ flist->low = 0;
- flist->high = -1;
- return;
- }
--- old/loadparm.c
+++ new/loadparm.c
-@@ -149,6 +149,7 @@ typedef struct
+@@ -153,6 +153,7 @@ typedef struct
int syslog_facility;
int timeout;
BOOL fake_super;
BOOL ignore_errors;
BOOL ignore_nonreadable;
-@@ -197,6 +198,7 @@ static service sDefault =
+@@ -201,6 +202,7 @@ static service sDefault =
/* syslog_facility; */ LOG_DAEMON,
/* timeout; */ 0,
/* fake_super; */ False,
/* ignore_errors; */ False,
/* ignore_nonreadable; */ False,
-@@ -313,6 +315,7 @@ static struct parm_struct parm_table[] =
+@@ -317,6 +319,7 @@ static struct parm_struct parm_table[] =
{"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0},
{"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0},
{"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0},
{"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0},
{"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0},
{"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0},
-@@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super)
+@@ -422,6 +425,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super)
FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
FN_LOCAL_BOOL(lp_list, list)
FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging)
--- old/options.c
+++ new/options.c
-@@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0;
+@@ -110,6 +110,7 @@ size_t bwlimit_writemax = 0;
int ignore_existing = 0;
int ignore_non_existing = 0;
int need_messages_from_generator = 0;
+int checksum_updating = 0;
- int max_delete = -1;
+ int max_delete = INT_MIN;
OFF_T max_size = 0;
OFF_T min_size = 0;
-@@ -302,6 +303,7 @@ void usage(enum logcode F)
+@@ -310,6 +311,7 @@ void usage(enum logcode F)
rprintf(F," -q, --quiet suppress non-error messages\n");
rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
rprintf(F," -r, --recursive recurse into directories\n");
-@@ -542,6 +544,7 @@ static struct poptOption long_options[]
+@@ -557,6 +559,7 @@ static struct poptOption long_options[]
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
-@@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg
+@@ -1954,7 +1957,9 @@ void server_options(char **args, int *ar
args[ac++] = basis_dir[i];
}
}
+ args[ac++] = "--checksum-updating";
+
- if (append_mode)
- args[ac++] = "--append";
+ if (append_mode) {
+ if (append_mode > 1)
--- old/rsync.h
+++ new/rsync.h
-@@ -589,6 +589,7 @@ extern int preserve_xattrs;
- #define EXTRA_LEN (sizeof (union file_extras))
- #define PTR_EXTRA_LEN ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN)
- #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN)
-+#define TIME_EXTRA_CNT ((SIZEOF_TIME_T + EXTRA_LEN - 1) / EXTRA_LEN)
-
- #define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx))
- #define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump))
-@@ -1070,6 +1071,12 @@ isDigit(const char *ptr)
- }
+@@ -662,6 +662,10 @@ extern int xattrs_ndx;
+ #define F_SUM(f) ((char*)OPT_EXTRA(f, LEN64_BUMP(f) + HLINK_BUMP(f) \
+ + SUM_EXTRA_CNT - 1))
- static inline int
-+isXDigit(const char *ptr)
-+{
-+ return isxdigit(*(unsigned char *)ptr);
-+}
++/* These are only valid on an entry read from a checksum file. */
++#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->num
++#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->num
+
-+static inline int
- isPrint(const char *ptr)
- {
- return isprint(*(unsigned char *)ptr);
+ /* Some utility defines: */
+ #define F_IS_ACTIVE(f) (f)->basename[0]
+ #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED)
--- old/rsync.yo
+++ new/rsync.yo
-@@ -307,6 +307,7 @@ to the detailed description below for a
+@@ -317,6 +317,7 @@ to the detailed description below for a
-q, --quiet suppress non-error messages
--no-motd suppress daemon-mode MOTD (see caveat)
-c, --checksum skip based on checksum, not mod-time & size
-a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
--no-OPTION turn off an implied OPTION (e.g. --no-D)
-r, --recursive recurse into directories
-@@ -502,9 +503,9 @@ uses a "quick check" that (by default) c
+@@ -515,9 +516,9 @@ uses a "quick check" that (by default) c
of last modification match between the sender and receiver. This option
changes this to compare a 128-bit MD4 checksum for each file that has a
matching size. Generating the checksums means that both sides will expend
The sending side generates its checksums while it is doing the file-system
scan that builds the list of the available files. The receiver generates
-@@ -512,12 +513,42 @@ its checksums when it is scanning for ch
+@@ -525,12 +526,42 @@ its checksums when it is scanning for ch
file that has the same size as the corresponding sender's file: files with
either a changed size or a changed checksum are selected for transfer.
everything (with -H being a notable omission).
--- old/rsyncd.conf.yo
+++ new/rsyncd.conf.yo
-@@ -198,6 +198,20 @@ locking on this file to ensure that the
+@@ -200,6 +200,20 @@ locking on this file to ensure that the
exceeded for the modules sharing the lock file.
The default is tt(/var/run/rsyncd.lock).
dit(bf(read only)) The "read only" option determines whether clients
will be able to upload files or not. If "read only" is true then any
attempted uploads will fail. If "read only" is false then uploads will
---- old/support/rsyncsums
-+++ new/support/rsyncsums
-@@ -0,0 +1,184 @@
-+#!/usr/bin/perl -w
-+use strict;
-+
-+use Getopt::Long;
-+use Cwd qw(abs_path cwd);
-+use Digest::MD4;
-+use Digest::MD5;
-+
-+our $SUMS_FILE = '.rsyncsums';
-+
-+our($recurse_opt, $force_reading, $help_opt);
-+our $verbosity = 0;
-+
-+&Getopt::Long::Configure('bundling');
-+&usage if !&GetOptions(
-+ 'recurse|r' => \$recurse_opt,
-+ 'force|f' => \$force_reading,
-+ 'verbose|v+' => \$verbosity,
-+ 'help|h' => \$help_opt,
-+) || $help_opt;
-+
-+my $start_dir = cwd();
-+
-+my @dirs = @ARGV;
-+@dirs = '.' unless @dirs;
-+foreach (@dirs) {
-+ $_ = abs_path($_);
-+}
-+
-+$| = 1;
-+
-+my $md4 = Digest::MD4->new;
-+my $md5 = Digest::MD5->new;
-+
-+while (@dirs) {
-+ my $dir = shift @dirs;
-+
-+ if (!chdir($dir)) {
-+ warn "Unable to chdir to $dir: $!\n";
-+ next;
-+ }
-+ if (!opendir(DP, '.')) {
-+ warn "Unable to opendir $dir: $!\n";
-+ next;
-+ }
-+
-+ if ($verbosity) {
-+ my $reldir = $dir;
-+ $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
-+ print "$reldir ... ";
-+ }
-+
-+ my $sums_mtime = (stat($SUMS_FILE))[9];
-+ my %cache;
-+ my @subdirs;
-+ my $cnt = 0;
-+ my $latest_time = 0;
-+ while (defined(my $fn = readdir(DP))) {
-+ next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn;
-+ if (-d _) {
-+ push(@subdirs, "$dir/$fn");
-+ next;
-+ }
-+ next unless -f _;
-+
-+ my($size,$mtime,$ctime) = (stat(_))[7,9,10];
-+
-+ $cache{$fn} = [ $size, $mtime, $ctime ];
-+ $cnt++;
-+
-+ $latest_time = $mtime if $mtime > $latest_time;
-+ $latest_time = $ctime if $ctime > $latest_time;
-+ }
-+
-+ closedir DP;
-+
-+ unshift(@dirs, sort @subdirs) if $recurse_opt;
-+
-+ if (!$cnt) {
-+ if (defined $sums_mtime) {
-+ print "(removed $SUMS_FILE) " if $verbosity;
-+ unlink($SUMS_FILE);
-+ }
-+ print "empty\n" if $verbosity;
-+ next;
-+ }
-+
-+ if (defined($sums_mtime) && $sums_mtime == $latest_time && !$force_reading) {
-+ print "OK\n" if $verbosity;
-+ next;
-+ }
-+
-+ if (open(FP, '+<', $SUMS_FILE)) {
-+ while (<FP>) {
-+ chomp;
-+ my($sum4, $sum5, $size, $mtime, $ctime, $fn) = split(' ', $_, 6);
-+ my $ref = $cache{$fn};
-+ if (defined $ref) {
-+ if ($$ref[0] == $size
-+ && $$ref[1] == $mtime && $$ref[2] == $ctime
-+ && $sum4 !~ /=/ && $sum5 !~ /=/) {
-+ $$ref[3] = $sum4;
-+ $$ref[4] = $sum5;
-+ $cnt--;
-+ } else {
-+ $$ref[3] = $$ref[4] = undef;
-+ }
-+ } else {
-+ $cnt = -1; # Force rewrite due to removed line.
-+ }
-+ }
-+ } else {
-+ open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n";
-+ $cnt = -1;
-+ }
-+
-+ if ($cnt) {
-+ print "updating\n" if $verbosity;
-+ while (my($fn, $ref) = each %cache) {
-+ next if defined $$ref[3] && defined $$ref[4];
-+ if (!open(IN, $fn)) {
-+ print STDERR "Unable to read $fn: $!\n";
-+ delete $cache{$fn};
-+ next;
-+ }
-+
-+ my($size,$mtime,$ctime) = (stat(IN))[7,9,10];
-+ my($sum4, $sum5);
-+
-+ while (1) {
-+ while (sysread(IN, $_, 64*1024)) {
-+ $md4->add($_);
-+ $md5->add($_);
-+ }
-+ $sum4 = $md4->hexdigest;
-+ $sum5 = $md5->hexdigest;
-+ print " $sum4 $sum5" if $verbosity > 2;
-+ print " $fn\n" if $verbosity > 1;
-+ my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10];
-+ last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
-+ $size = $size2;
-+ $mtime = $mtime2;
-+ $ctime = $ctime2;
-+ sysseek(IN, 0, 0);
-+ }
-+
-+ close IN;
-+
-+ $cache{$fn} = [ $size, $mtime, $ctime, $sum4, $sum5 ];
-+ }
-+
-+ $latest_time = 0;
-+ seek(FP, 0, 0);
-+ foreach my $fn (sort keys %cache) {
-+ my $ref = $cache{$fn};
-+ my($size, $mtime, $ctime, $sum4, $sum5) = @$ref;
-+ printf FP '%s %s %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn;
-+
-+ $latest_time = $mtime if $mtime > $latest_time;
-+ $latest_time = $ctime if $ctime > $latest_time;
-+ }
-+ truncate(FP, tell(FP));
-+ } else {
-+ print "OK.\n" if $verbosity;
-+ }
-+
-+ close FP;
-+
-+ utime $latest_time, $latest_time, $SUMS_FILE;
-+}
-+
-+sub usage
-+{
-+ die <<EOT;
-+Usage: rsyncsums [OPTIONS] [DIRS]
-+
-+Options:
-+ -r, --recurse Update $SUMS_FILE files in subdirectories too.
-+ -f, --force Force the reading of an $SUMS_FILE file that looks to be
-+ up-to-date. (Useful for weeding out old entries.)
-+ -v, --verbose Mention what we're doing. Repeat for more info.
-+ -h, --help Display this help message.
-+EOT
-+}
Matt McCutchen's Web Site / rsync / rsync-patches.git / blobdiff