./configure (optional if already run)
make
-TODO: when sending individual files (as opposed to an entire directory),
-we should still update the .rsyncsums file if we compute a new checksum.
-(The file is currently only written if we send an entire dir.)
-
--- old/clientserver.c
+++ new/clientserver.c
@@ -37,6 +37,7 @@ extern int sanitize_paths;
extern int io_timeout;
extern int no_detach;
extern int default_af_hint;
-@@ -634,6 +635,8 @@ static int rsync_module(int f_in, int f_
+@@ -641,6 +642,8 @@ static int rsync_module(int f_in, int f_
else if (am_root < 0) /* Treat --fake-super from client as --super. */
am_root = 2;
#define PTR_SIZE (sizeof (struct file_struct *))
-+#define FLAG_SUM_MISSING (1<<1)
-+#define FLAG_SUM_FOUND (1<<2)
++#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
++#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
+
int io_error;
int checksum_len;
dev_t filesystem_dev; /* used to implement -x */
-@@ -101,6 +106,9 @@ static char tmp_sum[MAX_DIGEST_LEN];
+@@ -101,6 +106,10 @@ static char tmp_sum[MAX_DIGEST_LEN];
static char empty_sum[MAX_DIGEST_LEN];
static int flist_count_offset; /* for --delete --progress */
static int dir_count = 0;
+static struct file_list *checksum_flist = NULL;
+static int checksum_matches = 0;
++static int checksum_updates = 0;
+static int regular_skipped = 0;
static void clean_flist(struct file_list *flist, int strip_root);
static void output_flist(struct file_list *flist);
-@@ -317,6 +325,275 @@ static void flist_done_allocating(struct
+@@ -317,6 +326,304 @@ static void flist_done_allocating(struct
flist->pool_boundary = ptr;
}
+ OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
+ }
+ file->dirname = dirname;
-+ bp = (char*)F_SUM(file);
++ bp = F_SUM(file);
+ memcpy(bp, sum, checksum_len);
++#if SIZEOF_TIME_T == 4
++ OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num = ctime;
++#else
+ memcpy(bp - SIZEOF_TIME_T, &ctime, SIZEOF_TIME_T);
++#endif
+
+ flist_expand(checksum_flist, 1);
+ checksum_flist->files[checksum_flist->count++] = file;
+ checksum_flist->sorted = checksum_flist->files;
+}
+
++static void write_checksums(const char *next_dirname, int whole_dir)
++{
++ static const char *dirname_save;
++ char fbuf[MAXPATHLEN];
++ const char *dirname;
++ int count, new_entries, counts_match, no_skipped;
++ time_t latest_time = 0;
++ FILE *out_fp;
++ int i;
++
++ dirname = dirname_save;
++ dirname_save = next_dirname;
++
++ if (!dirname)
++ return;
++
++ count = checksum_flist->count;
++ new_entries = checksum_updates != 0;
++ counts_match = count == checksum_matches;
++ no_skipped = whole_dir && regular_skipped == 0;
++
++ clean_flist(checksum_flist, 0);
++
++ checksum_flist->count = 0;
++ checksum_matches = 0;
++ checksum_updates = 0;
++ regular_skipped = 0;
++
++ if (dry_run)
++ return;
++
++ if (*dirname) {
++ if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf)
++ return;
++ } else
++ strlcpy(fbuf, ".rsyncsums", sizeof fbuf);
++
++ if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) {
++ unlink(fbuf);
++ return;
++ }
++
++ if (!new_entries && (counts_match || !whole_dir))
++ return;
++
++ if (!(out_fp = fopen(fbuf, "w")))
++ return;
++
++ new_entries = 0;
++ for (i = checksum_flist->low; i <= checksum_flist->high; i++) {
++ struct file_struct *file = checksum_flist->sorted[i];
++ const char *cp = F_SUM(file);
++ const char *end = cp + checksum_len;
++ const char *alt_sum = file->basename + strlen(file->basename) + 1;
++ time_t ctime;
++ if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
++ continue;
++#if SIZEOF_TIME_T == 4
++ ctime = OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num;
++#else
++ memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T);
++#endif
++ if (protocol_version >= 30)
++ fprintf(out_fp, "%s ", alt_sum);
++ if (file->flags & FLAG_SUM_MISSING) {
++ new_entries++;
++ do {
++ fprintf(out_fp, "==");
++ } while (++cp != end);
++ } else {
++ do {
++ fprintf(out_fp, "%02x", CVAL(cp, 0));
++ } while (++cp != end);
++ }
++ if (protocol_version < 30)
++ fprintf(out_fp, " %s", alt_sum);
++ if (*alt_sum == '=')
++ new_entries++;
++ fprintf(out_fp, " %10.0f %10.0f %10.0f %s\n",
++ (double)F_LENGTH(file), (double)file->modtime,
++ (double)ctime, file->basename);
++ if (file->modtime > ctime)
++ ctime = file->modtime;
++ if (ctime > latest_time)
++ latest_time = ctime;
++ }
++
++ fclose(out_fp);
++
++ if (whole_dir && new_entries == 0)
++ set_modtime(fbuf, latest_time, latest_time);
++ else
++ set_modtime(fbuf, latest_time-1, latest_time-1);
++}
++
+/* The direname value must remain unchanged during the lifespan of the
+ * created checksum_flist object because we use it directly. */
+static void read_checksums(const char *dirname)
+ char *cp;
+ FILE *fp;
+
++ write_checksums(dirname, 0);
++
+ if (checksum_flist) {
+ /* Reset the pool memory and empty the file-list array. */
+ pool_free_old(checksum_flist->file_pool,
+ checksum_flist->low = 0;
+ checksum_flist->high = -1;
+ checksum_matches = 0;
++ checksum_updates = 0;
+ regular_skipped = 0;
+
-+ if (dirname) {
-+ dlen = strlcpy(fbuf, dirname, sizeof fbuf);
-+ if (dlen >= (int)sizeof fbuf)
-+ return;
++ if (!dirname)
++ return;
++
++ dlen = strlcpy(fbuf, dirname, sizeof fbuf);
++ if (dlen >= (int)sizeof fbuf)
++ return;
++ if (dlen)
+ fbuf[dlen++] = '/';
-+ } else
-+ dlen = 0;
++ else
++ dirname = NULL;
+ strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen);
+ if (!(fp = fopen(fbuf, "r")))
+ return;
+
+ strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
+ if (is_excluded(fbuf, 0, ALL_FILTERS)) {
-+ flags |= FLAG_SUM_FOUND;
++ flags |= FLAG_SUM_KEEP;
+ checksum_matches++;
+ }
+
+
+ clean_flist(checksum_flist, 0);
+}
-+
-+static void write_checksums(const char *dirname)
-+{
-+ char fbuf[MAXPATHLEN];
-+ int count = checksum_flist->count;
-+ int new_entries = count > checksum_flist->high + 1;
-+ int counts_match = count == checksum_matches;
-+ int no_skipped = regular_skipped == 0;
-+ time_t latest_time = 0;
-+ FILE *out_fp;
-+ int i;
-+
-+ clean_flist(checksum_flist, 0);
-+
-+ checksum_flist->count = 0;
-+ checksum_matches = 0;
-+ regular_skipped = 0;
-+
-+ if (dry_run)
-+ return;
-+
-+ if (dirname) {
-+ if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf)
-+ return;
-+ } else
-+ strlcpy(fbuf, ".rsyncsums", sizeof fbuf);
-+
-+ if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) {
-+ unlink(fbuf);
-+ return;
-+ }
-+
-+ if (!new_entries && counts_match)
-+ return;
-+
-+ if (!(out_fp = fopen(fbuf, "w")))
-+ return;
-+
-+ for (i = checksum_flist->low; i <= checksum_flist->high; i++) {
-+ struct file_struct *file = checksum_flist->sorted[i];
-+ const char *cp = F_SUM(file);
-+ const char *end = cp + checksum_len;
-+ time_t ctime;
-+ if (!(file->flags & FLAG_SUM_FOUND))
-+ continue;
-+ memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T);
-+ if (protocol_version >= 30) {
-+ fprintf(out_fp, "%s ",
-+ file->basename + strlen(file->basename) + 1);
-+ }
-+ if (file->flags & FLAG_SUM_MISSING) {
-+ do {
-+ fprintf(out_fp, "==");
-+ } while (++cp != end);
-+ } else {
-+ do {
-+ fprintf(out_fp, "%02x", CVAL(cp, 0));
-+ } while (++cp != end);
-+ }
-+ if (protocol_version < 30) {
-+ fprintf(out_fp, " %s",
-+ file->basename + strlen(file->basename) + 1);
-+ }
-+ fprintf(out_fp, " %10.0f %10ld %10ld %s\n",
-+ (double)F_LENGTH(file), (long)file->modtime,
-+ (long)ctime, file->basename);
-+ if (file->modtime > ctime)
-+ ctime = file->modtime;
-+ if (ctime > latest_time)
-+ latest_time = ctime;
-+ }
-+
-+ fclose(out_fp);
-+
-+ set_modtime(fbuf, latest_time, latest_time);
-+}
+
int push_pathname(const char *dir, int len)
{
if (dir == pathname)
-@@ -973,34 +1250,24 @@ static struct file_struct *recv_file_ent
+@@ -980,34 +1287,24 @@ static struct file_struct *recv_file_ent
return file;
}
+ STRUCT_STAT st;
char *bp;
- if (strlcpy(thisname, fname, sizeof thisname)
-@@ -1077,6 +1344,8 @@ struct file_struct *make_file(const char
+ if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) {
+@@ -1086,6 +1383,8 @@ struct file_struct *make_file(const char
if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
if (ignore_perishable)
non_perishable_cnt++;
return NULL;
}
-@@ -1115,9 +1384,16 @@ struct file_struct *make_file(const char
+@@ -1124,9 +1423,16 @@ struct file_struct *make_file(const char
memcpy(lastdir, thisname, len);
lastdir[len] = '\0';
lastdir_len = len;
basename = thisname;
+ if (always_checksum && am_sender && flist && lastdir_len == -2) {
+ lastdir_len = -1;
-+ read_checksums(NULL);
++ read_checksums("");
+ }
+ }
basename_len = strlen(basename) + 1; /* count the '\0' */
#ifdef SUPPORT_LINKS
-@@ -1193,11 +1469,40 @@ struct file_struct *make_file(const char
+@@ -1202,11 +1508,44 @@ struct file_struct *make_file(const char
}
#endif
+ if (flist && (j = flist_find(checksum_flist, file)) >= 0) {
+ struct file_struct *fp = checksum_flist->sorted[j];
+ time_t ctime;
++#if SIZEOF_TIME_T == 4
++ ctime = OPT_EXTRA(fp, LEN64_BUMP(fp) + SUM_EXTRA_CNT)->num;
++#else
+ memcpy(&ctime, F_SUM(fp) - SIZEOF_TIME_T, SIZEOF_TIME_T);
++#endif
+ if (F_LENGTH(fp) == st.st_size
+ && fp->modtime == st.st_mtime && ctime == st.st_ctime) {
+ if (fp->flags & FLAG_SUM_MISSING) {
+ fp->flags &= ~FLAG_SUM_MISSING;
+ file_checksum(thisname, tmp_sum, st.st_size);
-+ memcpy((char*)F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
++ memcpy(F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
+ } else {
+ checksum_matches++;
+ memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
+ }
-+ fp->flags |= FLAG_SUM_FOUND;
++ fp->flags |= FLAG_SUM_KEEP;
+ } else {
+ clear_file(fp);
-+ goto compute_checksum;
++ goto compute_new_checksum;
+ }
+ } else {
-+ compute_checksum:
++ compute_new_checksum:
+ file_checksum(thisname, tmp_sum, st.st_size);
+ if (checksum_updating && flist) {
+ add_checksum(file->dirname, basename, basename_len,
+ st.st_size, st.st_mtime, st.st_ctime,
-+ tmp_sum, NULL, FLAG_SUM_FOUND);
++ tmp_sum, NULL, FLAG_SUM_KEEP);
+ }
+ }
+ }
/* This code is only used by the receiver when it is building
* a list of files for a delete pass. */
if (keep_dirlinks && linkname_len && flist) {
-@@ -1241,14 +1546,14 @@ void unmake_file(struct file_struct *fil
-
- static struct file_struct *send_file_name(int f, struct file_list *flist,
- char *fname, STRUCT_STAT *stp,
-- int flags, int filter_flags)
-+ int flags, int filter_level)
- {
- struct file_struct *file;
- #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS
- statx sx;
- #endif
-
-- file = make_file(fname, flist, stp, flags, filter_flags);
-+ file = make_file(fname, flist, stp, flags, filter_level);
- if (!file)
- return NULL;
-
-@@ -1442,7 +1747,7 @@ static void send_directory(int f, struct
- DIR *d;
- int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0;
- int start = flist->count;
-- int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
-+ int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS;
-
- assert(flist != NULL);
-
-@@ -1471,7 +1776,7 @@ static void send_directory(int f, struct
- continue;
- }
-
-- send_file_name(f, flist, fbuf, NULL, flags, filter_flags);
-+ send_file_name(f, flist, fbuf, NULL, flags, filter_level);
- }
-
- fbuf[len] = '\0';
-@@ -1483,6 +1788,9 @@ static void send_directory(int f, struct
+@@ -1499,6 +1838,9 @@ static void send_directory(int f, struct
closedir(d);
+ if (checksum_updating && always_checksum && am_sender && f >= 0)
-+ write_checksums(fbuf);
++ write_checksums(NULL, 1);
+
if (f >= 0 && recurse && !divert_dirs) {
- int i, end = flist->count - 1;
- /* send_if_directory() bumps flist->count, so use "end". */
-@@ -2206,7 +2514,7 @@ void flist_free(struct file_list *flist)
+ int i, end = flist->used - 1;
+ /* send_if_directory() bumps flist->used, so use "end". */
+@@ -1942,7 +2284,11 @@ struct file_list *send_file_list(int f,
+ * file-list to check if this is a 1-file xfer. */
+ send_extra_file_list(f, 1);
+ }
+- }
++ } else
++ flist_eof = 1;
++
++ if (checksum_updating && always_checksum && flist_eof)
++ read_checksums(NULL); /* writes any last updates */
+
+ return flist;
+ }
+@@ -2235,7 +2581,7 @@ void flist_free(struct file_list *flist)
if (!flist->prev || !flist_cnt)
pool_destroy(flist->file_pool);
pool_free_old(flist->file_pool, flist->pool_boundary);
if (flist->sorted && flist->sorted != flist->files)
-@@ -2225,6 +2533,7 @@ static void clean_flist(struct file_list
- if (!flist)
- return;
- if (flist->count == 0) {
-+ flist->low = 0;
- flist->high = -1;
- return;
- }
--- old/loadparm.c
+++ new/loadparm.c
-@@ -149,6 +149,7 @@ typedef struct
+@@ -152,6 +152,7 @@ typedef struct
int syslog_facility;
int timeout;
BOOL fake_super;
BOOL ignore_errors;
BOOL ignore_nonreadable;
-@@ -197,6 +198,7 @@ static service sDefault =
+@@ -200,6 +201,7 @@ static service sDefault =
/* syslog_facility; */ LOG_DAEMON,
/* timeout; */ 0,
/* fake_super; */ False,
/* ignore_errors; */ False,
/* ignore_nonreadable; */ False,
-@@ -313,6 +315,7 @@ static struct parm_struct parm_table[] =
+@@ -316,6 +318,7 @@ static struct parm_struct parm_table[] =
{"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0},
{"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0},
{"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0},
{"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0},
{"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0},
{"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0},
-@@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super)
+@@ -421,6 +424,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super)
FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
FN_LOCAL_BOOL(lp_list, list)
int ignore_non_existing = 0;
int need_messages_from_generator = 0;
+int checksum_updating = 0;
- int max_delete = -1;
+ int max_delete = INT_MIN;
OFF_T max_size = 0;
OFF_T min_size = 0;
-@@ -302,6 +303,7 @@ void usage(enum logcode F)
+@@ -303,6 +304,7 @@ void usage(enum logcode F)
rprintf(F," -q, --quiet suppress non-error messages\n");
rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
rprintf(F," -r, --recursive recurse into directories\n");
-@@ -542,6 +544,7 @@ static struct poptOption long_options[]
+@@ -544,6 +546,7 @@ static struct poptOption long_options[]
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
-@@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg
+@@ -1913,7 +1916,9 @@ void server_options(char **args,int *arg
args[ac++] = basis_dir[i];
}
}
args[ac++] = "--append";
--- old/rsync.h
+++ new/rsync.h
-@@ -589,6 +589,7 @@ extern int preserve_xattrs;
- #define EXTRA_LEN (sizeof (union file_extras))
- #define PTR_EXTRA_LEN ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN)
+@@ -592,6 +592,7 @@ extern int xattrs_ndx;
+ #define DEV_EXTRA_CNT 2
+ #define DIRNODE_EXTRA_CNT 3
#define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN)
+#define TIME_EXTRA_CNT ((SIZEOF_TIME_T + EXTRA_LEN - 1) / EXTRA_LEN)
#define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx))
#define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump))
-@@ -1070,6 +1071,12 @@ isDigit(const char *ptr)
+@@ -1077,6 +1078,12 @@ isDigit(const char *ptr)
}
static inline int
return isprint(*(unsigned char *)ptr);
--- old/rsync.yo
+++ new/rsync.yo
-@@ -307,6 +307,7 @@ to the detailed description below for a
+@@ -322,6 +322,7 @@ to the detailed description below for a
-q, --quiet suppress non-error messages
--no-motd suppress daemon-mode MOTD (see caveat)
-c, --checksum skip based on checksum, not mod-time & size
-a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
--no-OPTION turn off an implied OPTION (e.g. --no-D)
-r, --recursive recurse into directories
-@@ -502,9 +503,9 @@ uses a "quick check" that (by default) c
+@@ -518,9 +519,9 @@ uses a "quick check" that (by default) c
of last modification match between the sender and receiver. This option
changes this to compare a 128-bit MD4 checksum for each file that has a
matching size. Generating the checksums means that both sides will expend
The sending side generates its checksums while it is doing the file-system
scan that builds the list of the available files. The receiver generates
-@@ -512,12 +513,42 @@ its checksums when it is scanning for ch
+@@ -528,12 +529,42 @@ its checksums when it is scanning for ch
file that has the same size as the corresponding sender's file: files with
either a changed size or a changed checksum are selected for transfer.
+
+ my $sums_mtime = (stat($SUMS_FILE))[9];
+ my %cache;
-+ my @fp;
+ my @subdirs;
+ my $cnt = 0;
+ my $latest_time = 0;
+ }
+ $sum4 = $md4->hexdigest;
+ $sum5 = $md5->hexdigest;
-+ print " $sum4 $sum5 $fn\n" if $verbosity > 1;
++ print " $sum4 $sum5" if $verbosity > 2;
++ print " $fn\n" if $verbosity > 1;
+ my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10];
+ last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
+ $size = $size2;
+ foreach my $fn (sort keys %cache) {
+ my $ref = $cache{$fn};
+ my($size, $mtime, $ctime, $sum4, $sum5) = @$ref;
-+ printf FP '%s %s %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn;
++ printf FP '%s %s %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn;
+
+ $latest_time = $mtime if $mtime > $latest_time;
+ $latest_time = $ctime if $ctime > $latest_time;