Renamed flags.diff fileflags.diff.

[rsync/rsync-patches.git] / checksum-updating.diff

This builds on the sender optimization feature of the checksum-reading
patch and adds the ability to create and/or updates the .rsyncsums files
when --checksum-updating (or "checksum updating = true") is specified.

To use this patch, run these commands for a successful build:

    patch -p1 <patches/checksum-reading.diff
    patch -p1 <patches/checksum-updating.diff
    ./configure                               (optional if already run)
    make

diff --git a/clientserver.c b/clientserver.c
--- a/clientserver.c
+++ b/clientserver.c
@@ -39,6 +39,7 @@ extern int numeric_ids;
 extern int filesfrom_fd;
 extern int remote_protocol;
 extern int protocol_version;
+extern int checksum_updating;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
@@ -746,6 +747,8 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 	else if (am_root < 0) /* Treat --fake-super from client as --super. */
 		am_root = 2;
 
+	checksum_updating = lp_checksum_updating(i);
+
 	if (filesfrom_fd == 0)
 		filesfrom_fd = f_in;
 
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -26,6 +26,7 @@
 #include "io.h"
 
 extern int verbose;
+extern int dry_run;
 extern int am_root;
 extern int am_server;
 extern int am_daemon;
@@ -58,6 +59,7 @@ extern int implied_dirs;
 extern int file_extra_cnt;
 extern int ignore_perishable;
 extern int non_perishable_cnt;
+extern int checksum_updating;
 extern int prune_empty_dirs;
 extern int copy_links;
 extern int copy_unsafe_links;
@@ -83,6 +85,9 @@ extern iconv_t ic_send, ic_recv;
 
 #define PTR_SIZE (sizeof (struct file_struct *))
 
+#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */
+#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */
+
 int io_error;
 int checksum_len;
 dev_t filesystem_dev; /* used to implement -x */
@@ -121,6 +126,9 @@ static char tmp_sum[MAX_DIGEST_LEN];
 static char empty_sum[MAX_DIGEST_LEN];
 static int flist_count_offset; /* for --delete --progress */
 static int dir_count = 0;
+static int checksum_matches = 0;
+static int checksum_updates = 0;
+static int regular_skipped = 0;
 static struct file_list *checksum_flist = NULL;
 
 static void flist_sort_and_clean(struct file_list *flist, int strip_root);
@@ -316,7 +324,8 @@ static void flist_done_allocating(struct file_list *flist)
 
 /* The len count is the length of the basename + 1 for the null. */
 static int add_checksum(const char *dirname, const char *basename, int len,
-			OFF_T file_length, time_t mtime, const char *sum)
+			OFF_T file_length, time_t mtime, int32 ctime, int32 inode,
+			const char *sum, const char *alt_sum, int flags)
 {
 	struct file_struct *file;
 	int alloc_len, extra_len;
@@ -327,13 +336,14 @@ static int add_checksum(const char *dirname, const char *basename, int len,
 	if (file_length == 0)
 		return 0;
 
-	extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT)
+	/* "2" is for a 32-bit ctime num and an 32-bit inode num. */
+	extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2)
 		  * EXTRA_LEN;
 #if EXTRA_ROUNDING > 0
 	if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
 		extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
 #endif
-	alloc_len = FILE_STRUCT_LEN + extra_len + len;
+	alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1;
 	bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum");
 
 	memset(bp, 0, extra_len + FILE_STRUCT_LEN);
@@ -342,7 +352,14 @@ static int add_checksum(const char *dirname, const char *basename, int len,
 	bp += FILE_STRUCT_LEN;
 
 	memcpy(bp, basename, len);
+	if (alt_sum)
+		strlcpy(bp+len, alt_sum, checksum_len*2 + 1);
+	else {
+		memset(bp+len, '=', checksum_len*2);
+		bp[len+checksum_len*2] = '\0';
+	}
 
+	file->flags = flags;
 	file->mode = S_IFREG;
 	file->modtime = mtime;
 	file->len32 = (uint32)file_length;
@@ -353,6 +370,8 @@ static int add_checksum(const char *dirname, const char *basename, int len,
 	file->dirname = dirname;
 	bp = F_SUM(file);
 	memcpy(bp, sum, checksum_len);
+	F_CTIME(file) = ctime;
+	F_INODE(file) = inode;
 
 	flist_expand(checksum_flist, 1);
 	checksum_flist->files[checksum_flist->used++] = file;
@@ -362,17 +381,104 @@ static int add_checksum(const char *dirname, const char *basename, int len,
 	return 1;
 }
 
+static void write_checksums(const char *next_dirname, int whole_dir)
+{
+	static const char *dirname_save;
+	char fbuf[MAXPATHLEN];
+	const char *dirname;
+	int used, new_entries, counts_match, no_skipped;
+	FILE *out_fp;
+	int i;
+
+	dirname = dirname_save;
+	dirname_save = next_dirname;
+
+	if (!dirname)
+		return;
+
+	used = checksum_flist->used;
+	new_entries = checksum_updates != 0;
+	counts_match = used == checksum_matches;
+	no_skipped = whole_dir && regular_skipped == 0;
+
+	flist_sort_and_clean(checksum_flist, 0);
+
+	checksum_flist->used = 0;
+	checksum_matches = 0;
+	checksum_updates = 0;
+	regular_skipped = 0;
+
+	if (dry_run)
+		return;
+
+	if (*dirname) {
+		if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf)
+			return;
+	} else
+		strlcpy(fbuf, ".rsyncsums", sizeof fbuf);
+
+	if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) {
+		unlink(fbuf);
+		return;
+	}
+
+	if (!new_entries && (counts_match || !whole_dir))
+		return;
+
+	if (!(out_fp = fopen(fbuf, "w")))
+		return;
+
+	new_entries = 0;
+	for (i = checksum_flist->low; i <= checksum_flist->high; i++) {
+		struct file_struct *file = checksum_flist->sorted[i];
+		const char *cp = F_SUM(file);
+		const char *end = cp + checksum_len;
+		const char *alt_sum = file->basename + strlen(file->basename) + 1;
+		int32 ctime, inode;
+		if (whole_dir && !(file->flags & FLAG_SUM_KEEP))
+			continue;
+		ctime = F_CTIME(file);
+		inode = F_INODE(file);
+		if (protocol_version >= 30)
+			fprintf(out_fp, "%s ", alt_sum);
+		if (file->flags & FLAG_SUM_MISSING) {
+			new_entries++;
+			do {
+				fprintf(out_fp, "==");
+			} while (++cp != end);
+		} else {
+			do {
+				fprintf(out_fp, "%02x", (int)CVAL(cp, 0));
+			} while (++cp != end);
+		}
+		if (protocol_version < 30)
+			fprintf(out_fp, " %s", alt_sum);
+		if (*alt_sum == '=')
+			new_entries++;
+		fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n",
+			(double)F_LENGTH(file), (double)file->modtime,
+			(long)ctime, (long)inode, file->basename);
+	}
+
+	fclose(out_fp);
+}
+
 /* The direname value must remain unchanged during the lifespan of the
  * created checksum_flist object because we use it directly. */
 static void read_checksums(const char *dirname)
 {
 	char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
+	const char *alt_sum = NULL;
 	OFF_T file_length;
 	time_t mtime;
-	int len, dlen, i;
+	int32 ctime, inode;
+	int len, dlen, i, flags;
 	char *cp;
 	FILE *fp;
 
+	if (checksum_updating)
+		write_checksums(dirname, 0);
+
 	if (checksum_flist) {
 		/* Reset the pool memory and empty the file-list array. */
 		pool_free_old(checksum_flist->file_pool,
@@ -383,6 +489,9 @@ static void read_checksums(const char *dirname)
 
 	checksum_flist->low = 0;
 	checksum_flist->high = -1;
+	checksum_matches = 0;
+	checksum_updates = 0;
+	regular_skipped = 0;
 
 	if (!dirname)
 		return;
@@ -401,7 +510,7 @@ static void read_checksums(const char *dirname)
 	while (fgets(line, sizeof line, fp)) {
 		cp = line;
 		if (protocol_version >= 30) {
-			char *alt_sum = cp;
+			alt_sum = cp;
 			if (*cp == '=')
 				while (*++cp == '=') {}
 			else
@@ -412,7 +521,14 @@ static void read_checksums(const char *dirname)
 		}
 
 		if (*cp == '=') {
-			continue;
+			for (i = 0; i < checksum_len*2; i++, cp++) {
+				if (*cp != '=') {
+					cp = "";
+					break;
+				}
+			}
+			memset(sum, 0, checksum_len);
+			flags = FLAG_SUM_MISSING;
 		} else {
 			for (i = 0; i < checksum_len*2; i++, cp++) {
 				int x;
@@ -430,13 +546,14 @@ static void read_checksums(const char *dirname)
 				else
 					sum[i/2] = x << 4;
 			}
+			flags = 0;
 		}
 		if (*cp != ' ')
 			break;
 		while (*++cp == ' ') {}
 
 		if (protocol_version < 30) {
-			char *alt_sum = cp;
+			alt_sum = cp;
 			if (*cp == '=')
 				while (*++cp == '=') {}
 			else
@@ -460,16 +577,16 @@ static void read_checksums(const char *dirname)
 			break;
 		while (*++cp == ' ') {}
 
-		/* Ignore ctime. */
+		ctime = 0;
 		while (isDigit(cp))
-			cp++;
+			ctime = ctime * 10 + *cp++ - '0';
 		if (*cp != ' ')
 			break;
 		while (*++cp == ' ') {}
 
-		/* Ignore inode. */
+		inode = 0;
 		while (isDigit(cp))
-			cp++;
+			inode = inode * 10 + *cp++ - '0';
 		if (*cp != ' ')
 			break;
 		while (*++cp == ' ') {}
@@ -486,8 +603,13 @@ static void read_checksums(const char *dirname)
 			continue;
 
 		strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
+		if (is_excluded(fbuf, 0, ALL_FILTERS)) {
+			flags |= FLAG_SUM_KEEP;
+			checksum_matches++;
+		}
 
-		add_checksum(dirname, cp, len, file_length, mtime, sum);
+		add_checksum(dirname, cp, len, file_length, mtime, ctime, inode,
+			     sum, alt_sum, flags);
 	}
 	fclose(fp);
 
@@ -1278,6 +1400,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) {
 		if (ignore_perishable)
 			non_perishable_cnt++;
+		if (S_ISREG(st.st_mode))
+			regular_skipped++;
 		return NULL;
 	}
 
@@ -1408,13 +1532,36 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		int j;
 		if (flist && (j = flist_find(checksum_flist, file)) >= 0) {
 			struct file_struct *fp = checksum_flist->sorted[j];
+			int32 ctime = F_CTIME(fp);
+			int32 inode = F_INODE(fp);
 			if (F_LENGTH(fp) == st.st_size
-			 && fp->modtime == st.st_mtime)
-				memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
-			else
-				file_checksum(thisname, tmp_sum, st.st_size);
-		} else
+			 && fp->modtime == st.st_mtime
+			 && ctime == (int32)st.st_ctime
+			 && inode == (int32)st.st_ino) {
+				if (fp->flags & FLAG_SUM_MISSING) {
+					fp->flags &= ~FLAG_SUM_MISSING;
+					checksum_updates++;
+					file_checksum(thisname, tmp_sum, st.st_size);
+					memcpy(F_SUM(fp), tmp_sum, MAX_DIGEST_LEN);
+				} else {
+					checksum_matches++;
+					memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN);
+				}
+				fp->flags |= FLAG_SUM_KEEP;
+			} else {
+				clear_file(fp);
+				goto compute_new_checksum;
+			}
+		} else {
+		  compute_new_checksum:
 			file_checksum(thisname, tmp_sum, st.st_size);
+			if (checksum_updating && flist) {
+				checksum_updates +=
+				    add_checksum(file->dirname, basename, basename_len,
+						 st.st_size, st.st_mtime, st.st_ctime,
+						 st.st_ino, tmp_sum, NULL, FLAG_SUM_KEEP);
+			}
+		}
 	}
 
 	/* This code is only used by the receiver when it is building
@@ -1709,6 +1856,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len,
 
 	closedir(d);
 
+	if (checksum_updating && always_checksum && am_sender && f >= 0)
+		write_checksums(NULL, 1);
+
 	if (f >= 0 && recurse && !divert_dirs) {
 		int i, end = flist->used - 1;
 		/* send_if_directory() bumps flist->used, so use "end". */
@@ -2274,6 +2424,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 		}
 	} else
 		flist_eof = 1;
+	
+	if (checksum_updating && always_checksum && flist_eof)
+		read_checksums(NULL); /* writes any last updates */
 
 	return flist;
 }
diff --git a/loadparm.c b/loadparm.c
--- a/loadparm.c
+++ b/loadparm.c
@@ -154,6 +154,7 @@ typedef struct
 	int syslog_facility;
 	int timeout;
 
+	BOOL checksum_updating;
 	BOOL fake_super;
 	BOOL ignore_errors;
 	BOOL ignore_nonreadable;
@@ -205,6 +206,7 @@ static service sDefault =
  /* syslog_facility; */		LOG_DAEMON,
  /* timeout; */			0,
 
+ /* checksum_updating; */	False,
  /* fake_super; */		False,
  /* ignore_errors; */		False,
  /* ignore_nonreadable; */	False,
@@ -306,6 +308,7 @@ static struct parm_struct parm_table[] =
 
  {"auth users",        P_STRING, P_LOCAL, &sDefault.auth_users,        NULL,0},
  {"charset",           P_STRING, P_LOCAL, &sDefault.charset,           NULL,0},
+ {"checksum updating", P_BOOL,   P_LOCAL, &sDefault.checksum_updating, NULL,0},
  {"comment",           P_STRING, P_LOCAL, &sDefault.comment,           NULL,0},
  {"dont compress",     P_STRING, P_LOCAL, &sDefault.dont_compress,     NULL,0},
  {"exclude from",      P_STRING, P_LOCAL, &sDefault.exclude_from,      NULL,0},
@@ -428,6 +431,7 @@ FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
 FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
 FN_LOCAL_INTEGER(lp_timeout, timeout)
 
+FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating)
 FN_LOCAL_BOOL(lp_fake_super, fake_super)
 FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
 FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable)
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -112,6 +112,7 @@ size_t bwlimit_writemax = 0;
 int ignore_existing = 0;
 int ignore_non_existing = 0;
 int need_messages_from_generator = 0;
+int checksum_updating = 0;
 int max_delete = INT_MIN;
 OFF_T max_size = 0;
 OFF_T min_size = 0;
@@ -316,6 +317,7 @@ void usage(enum logcode F)
   rprintf(F," -q, --quiet                 suppress non-error messages\n");
   rprintf(F,"     --no-motd               suppress daemon-mode MOTD (see manpage caveat)\n");
   rprintf(F," -c, --checksum              skip based on checksum, not mod-time & size\n");
+  rprintf(F,"     --checksum-updating     sender updates .rsyncsums files\n");
   rprintf(F," -a, --archive               archive mode; equals -rlptgoD (no -H,-A,-X)\n");
   rprintf(F,"     --no-OPTION             turn off an implied OPTION (e.g. --no-D)\n");
   rprintf(F," -r, --recursive             recurse into directories\n");
@@ -566,6 +568,7 @@ static struct poptOption long_options[] = {
   {"checksum",        'c', POPT_ARG_VAL,    &always_checksum, 1, 0, 0 },
   {"no-checksum",      0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
   {"no-c",             0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
+  {"checksum-updating",0,  POPT_ARG_NONE,   &checksum_updating, 0, 0, 0 },
   {"block-size",      'B', POPT_ARG_LONG,   &block_size, 0, 0, 0 },
   {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
   {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -1988,7 +1991,9 @@ void server_options(char **args, int *argc_p)
 				args[ac++] = basis_dir[i];
 			}
 		}
-	}
+	} else if (checksum_updating)
+		args[ac++] = "--checksum-updating";
+
 
 	if (append_mode) {
 		if (append_mode > 1)
diff --git a/rsync.h b/rsync.h
--- a/rsync.h
+++ b/rsync.h
@@ -680,6 +680,10 @@ extern int xattrs_ndx;
 #define F_SUM(f) ((char*)OPT_EXTRA(f, LEN64_BUMP(f) + HLINK_BUMP(f) \
 				    + SUM_EXTRA_CNT - 1))
 
+/* These are only valid on an entry read from a checksum file. */
+#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->num
+#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->num
+
 /* Some utility defines: */
 #define F_IS_ACTIVE(f) (f)->basename[0]
 #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED)
diff --git a/rsync.yo b/rsync.yo
--- a/rsync.yo
+++ b/rsync.yo
@@ -317,6 +317,7 @@ to the detailed description below for a complete description.  verb(
  -q, --quiet                 suppress non-error messages
      --no-motd               suppress daemon-mode MOTD (see caveat)
  -c, --checksum              skip based on checksum, not mod-time & size
+     --checksum-updating     sender updates .rsyncsums files
  -a, --archive               archive mode; equals -rlptgoD (no -H,-A,-X)
      --no-OPTION             turn off an implied OPTION (e.g. --no-D)
  -r, --recursive             recurse into directories
@@ -516,9 +517,9 @@ uses a "quick check" that (by default) checks if each file's size and time
 of last modification match between the sender and receiver.  This option
 changes this to compare a 128-bit MD4 checksum for each file that has a
 matching size.  Generating the checksums means that both sides will expend
-a lot of disk I/O reading all the data in the files in the transfer (and
-this is prior to any reading that will be done to transfer changed files),
-so this can slow things down significantly.
+a lot of disk I/O reading the data in all the files in the transfer, so
+this can slow things down significantly (and this is prior to any reading
+that will be done to transfer the files that have changed).
 
 The sending side generates its checksums while it is doing the file-system
 scan that builds the list of the available files.  The receiver generates
@@ -526,12 +527,42 @@ its checksums when it is scanning for changed files, and will checksum any
 file that has the same size as the corresponding sender's file:  files with
 either a changed size or a changed checksum are selected for transfer.
 
+Starting with version 3.0.0, the sending side will look for a checksum
+summary file and use a pre-generated checksum that it reads out of the file
+(as long as it matches the file's size and modified time).  This allows a
+server to support the --checksum option to clients without having to
+recompute the checksums for each client.  See the bf(--checksum-updating)
+option for a way to have rsync create/update these checksum files.
+
 Note that rsync always verifies that each em(transferred) file was
 correctly reconstructed on the receiving side by checking a whole-file
 checksum that is generated as the file is transferred, but that
 automatic after-the-transfer verification has nothing to do with this
 option's before-the-transfer "Does this file need to be updated?" check.
 
+dit(bf(--checksum-updating)) This option tells the sending side to create
+and/or update per-directory checksum files that are used by the
+bf(--checksum) option.  The file that is updated is named .rsyncsums.  If
+pre-transfer checksums are not being computed, this option has no effect.
+
+The checksum files stores the computed checksum, last-known size,
+modification time, and name for each file in the current directory.  If a
+later transfer finds that a file matches its prior size and modification
+time, the checksum is assumed to still be correct.  Otherwise it is
+recomputed and udpated in the file.
+
+To avoid transferring the system's checksum files, you can use an exclude
+(e.g. bf(--exclude=.rsyncsums)).  To make this easier to type, you can use
+a popt alias.  For instance, adding the following line in your ~/.popt file
+defines a bf(-cc) option that enables checksum updating and excludes the
+checksum files:
+
+verb(  rsync alias --cc --checksum-updating --exclude=.rsyncsums)
+
+An rsync daemon does not allow the client to control this setting, so see
+the "checksum updating" daemon config option for information on how to make
+a daemon maintain these checksum files.
+
 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
 way of saying you want recursion and want to preserve almost
 everything (with -H being a notable omission).
diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
--- a/rsyncd.conf.yo
+++ b/rsyncd.conf.yo
@@ -281,6 +281,20 @@ locking on this file to ensure that the max connections limit is not
 exceeded for the modules sharing the lock file.
 The default is tt(/var/run/rsyncd.lock).
 
+dit(bf(checksum updating)) This option tells rsync to update/create the
+checksum information in the per-directory checksum files when users copy
+files using the bf(--checksum) option.  Any file that has changed since it
+was last checksummed (or is not mentioned) has its data updated in the
+.rsyncsums file.
+
+Note that this updating will occur even if the module is listed as being
+read-only.  If you want to hide these files (and you will almost always
+want to do), add ".rsyncsums" to the module's exclude setting.
+
+Note also that the client's command-line option, bf(--checksum-updating),
+has no effect on a daemon.  A daemon will only update/create checksum files
+if this config option is true.
+
 dit(bf(read only)) The "read only" option determines whether clients
 will be able to upload files or not. If "read only" is true then any
 attempted uploads will fail. If "read only" is false then uploads will