Use "use warnings" rather than -w on the #! line.

[rsync/rsync-patches.git] / log-checksum.diff

This patch to rsync adds a %C log escape that expands to the sender's
post-transfer checksum of a file for protocol 30 or above.  This way, if
you need the MD5 checksums of transferred files, you can have rsync log
them instead of spending extra processor time on a separate command to
compute them.

-- Matt McCutchen <hashproduct@gmail.com>

To use this patch, run these commands for a successful build:

    patch -p1 <patches/log-checksum.diff
    ./configure                                 (optional if already run)
    make

diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -68,6 +68,7 @@ extern int sanitize_paths;
 extern int munge_symlinks;
 extern int need_unsorted_flist;
 extern int sender_symlink_iconv;
+extern int sender_keeps_checksum;
 extern int unsort_ndx;
 extern struct stats stats;
 extern char *filesfrom_host;
@@ -1223,6 +1224,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		extra_len += EXTRA_LEN;
 #endif
 
+	if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
+		file_checksum(thisname, tmp_sum, st.st_size);
+		if (sender_keeps_checksum)
+			extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
+	}
+
 #if EXTRA_ROUNDING > 0
 	if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
 		extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
@@ -1286,9 +1293,6 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		memcpy(bp + basename_len, linkname, linkname_len);
 #endif
 
-	if (always_checksum && am_sender && S_ISREG(st.st_mode))
-		file_checksum(thisname, tmp_sum, st.st_size);
-
 	if (am_sender)
 		F_PATHNAME(file) = pathname;
 	else if (!pool)
@@ -1300,6 +1304,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		return NULL;
 	}
 
+	if (sender_keeps_checksum && S_ISREG(st.st_mode))
+		memcpy(F_SUM(file), tmp_sum, checksum_len);
+
 	if (unsort_ndx)
 		F_NDX(file) = dir_count;
 
diff --git a/log.c b/log.c
--- a/log.c
+++ b/log.c
@@ -32,8 +32,10 @@ extern int local_server;
 extern int quiet;
 extern int module_id;
 extern int msg_fd_out;
+extern int checksum_len;
 extern int allow_8bit_chars;
 extern int protocol_version;
+extern int always_checksum;
 extern int preserve_times;
 extern int uid_ndx;
 extern int gid_ndx;
@@ -57,6 +59,7 @@ extern iconv_t ic_send, ic_recv;
 extern char curr_dir[];
 extern char *full_module_path;
 extern unsigned int module_dirlen;
+extern char sender_file_sum[MAX_DIGEST_LEN];
 
 static int log_initialised;
 static int logfile_was_closed;
@@ -632,6 +635,28 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 			snprintf(buf2, sizeof buf2, fmt, (double)b);
 			n = buf2;
 			break;
+		case 'C':
+			if (protocol_version >= 30
+			 && (iflags & ITEM_TRANSFER
+			  || (always_checksum && S_ISREG(file->mode)))) {
+				int i, x1, x2;
+				const char *sum = iflags & ITEM_TRANSFER
+						? sender_file_sum : F_SUM(file);
+				c = buf2 + checksum_len*2;
+				*c = '\0';
+				for (i = checksum_len; --i >= 0; ) {
+					x1 = CVAL(sum, i);
+					x2 = x1 >> 4;
+					x1 &= 0xF;
+					*--c = x1 <= 9 ? x1 + '0' : x1 + 'a' - 10;
+					*--c = x2 <= 9 ? x2 + '0' : x2 + 'a' - 10;
+				}
+			} else {
+				memset(buf2, ' ', checksum_len*2);
+				buf2[checksum_len*2] = '\0';
+			}
+			n = buf2;
+			break;
 		case 'i':
 			if (iflags & ITEM_DELETED) {
 				n = "*deleting  ";
diff --git a/main.c b/main.c
--- a/main.c
+++ b/main.c
@@ -37,6 +37,7 @@ extern int am_generator;
 extern int am_daemon;
 extern int inc_recurse;
 extern int blocking_io;
+extern int always_checksum;
 extern int remove_source_files;
 extern int need_messages_from_generator;
 extern int kluge_around_eof;
@@ -68,6 +69,8 @@ extern int connect_timeout;
 extern pid_t cleanup_child_pid;
 extern unsigned int module_dirlen;
 extern struct stats stats;
+extern char *stdout_format;
+extern char *logfile_format;
 extern char *filesfrom_host;
 extern char *partial_dir;
 extern char *dest_option;
@@ -85,6 +88,7 @@ int local_server = 0;
 int daemon_over_rsh = 0;
 mode_t orig_umask = 0;
 int batch_gen_fd = -1;
+int sender_keeps_checksum = 0;
 
 /* There's probably never more than at most 2 outstanding child processes,
  * but set it higher, just in case. */
@@ -1003,6 +1007,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 
 	if (am_sender) {
 		keep_dirlinks = 0; /* Must be disabled on the sender. */
+
+		if (always_checksum
+		 && (log_format_has(stdout_format, 'C')
+		  || log_format_has(logfile_format, 'C')))
+			sender_keeps_checksum = 1;
+
 		if (protocol_version >= 30)
 			io_start_multiplex_out();
 		else
diff --git a/match.c b/match.c
--- a/match.c
+++ b/match.c
@@ -25,8 +25,10 @@ extern int verbose;
 extern int do_progress;
 extern int checksum_seed;
 extern int append_mode;
+extern int checksum_len;
 
 int updating_basis_file;
+char sender_file_sum[MAX_DIGEST_LEN];
 
 static int false_alarms;
 static int hash_hits;
@@ -329,9 +331,6 @@ static void hash_search(int f,struct sum_struct *s,
  **/
 void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
 {
-	char file_sum[MAX_DIGEST_LEN];
-	int sum_len;
-
 	last_match = 0;
 	false_alarms = 0;
 	hash_hits = 0;
@@ -379,18 +378,28 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
 		matched(f, s, buf, len, -1);
 	}
 
-	sum_len = sum_end(file_sum);
-	/* If we had a read error, send a bad checksum. */
-	if (buf && buf->status != 0)
-		file_sum[0]++;
+	if (sum_end(sender_file_sum) != checksum_len)
+		overflow_exit("checksum_len"); /* Impossible... */
+
+	/* If we had a read error, send a bad checksum.  We use all bits
+	 * off as long as the checksum doesn't happen to be that, in
+	 * which case we turn the last 0 bit into a 1. */
+	if (buf && buf->status != 0) {
+		int i;
+		for (i = 0; i < checksum_len && sender_file_sum[i] == 0; i++) {}
+		memset(sender_file_sum, 0, checksum_len);
+		if (i == checksum_len)
+			sender_file_sum[i-1]++;
+	}
 
 	if (verbose > 2)
 		rprintf(FINFO,"sending file_sum\n");
-	write_buf(f, file_sum, sum_len);
+	write_buf(f, sender_file_sum, checksum_len);
 
-	if (verbose > 2)
+	if (verbose > 2) {
 		rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n",
 			false_alarms, hash_hits, matches);
+	}
 
 	total_hash_hits += hash_hits;
 	total_false_alarms += false_alarms;
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -1525,7 +1525,8 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 		else if (log_format_has(stdout_format, 'i'))
 			stdout_format_has_i = itemize_changes | 1;
 		if (!log_format_has(stdout_format, 'b')
-		 && !log_format_has(stdout_format, 'c'))
+		 && !log_format_has(stdout_format, 'c')
+		 && !log_format_has(stdout_format, 'C'))
 			log_before_transfer = !am_server;
 	} else if (itemize_changes) {
 		stdout_format = "%i %n%L";
diff --git a/receiver.c b/receiver.c
--- a/receiver.c
+++ b/receiver.c
@@ -46,6 +46,7 @@ extern int remove_source_files;
 extern int append_mode;
 extern int sparse_files;
 extern int keep_partial;
+extern int checksum_len;
 extern int checksum_seed;
 extern int inplace;
 extern int delay_updates;
@@ -54,6 +55,7 @@ extern struct stats stats;
 extern char *tmpdir;
 extern char *partial_dir;
 extern char *basis_dir[];
+extern char sender_file_sum[MAX_DIGEST_LEN];
 extern struct file_list *cur_flist, *first_flist, *dir_flist;
 extern struct filter_list_struct daemon_filter_list;
 
@@ -166,10 +168,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 			const char *fname, int fd, OFF_T total_size)
 {
 	static char file_sum1[MAX_DIGEST_LEN];
-	static char file_sum2[MAX_DIGEST_LEN];
 	struct map_struct *mapbuf;
 	struct sum_struct sum;
-	int32 len, sum_len;
+	int32 len;
 	OFF_T offset = 0;
 	OFF_T offset2;
 	char *data;
@@ -302,15 +303,16 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 		exit_cleanup(RERR_FILEIO);
 	}
 
-	sum_len = sum_end(file_sum1);
+	if (sum_end(file_sum1) != checksum_len)
+		overflow_exit("checksum_len"); /* Impossible... */
 
 	if (mapbuf)
 		unmap_file(mapbuf);
 
-	read_buf(f_in, file_sum2, sum_len);
+	read_buf(f_in, sender_file_sum, checksum_len);
 	if (verbose > 2)
 		rprintf(FINFO,"got file_sum\n");
-	if (fd != -1 && memcmp(file_sum1, file_sum2, sum_len) != 0)
+	if (fd != -1 && memcmp(file_sum1, sender_file_sum, checksum_len) != 0)
 		return 0;
 	return 1;
 }
diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
--- a/rsyncd.conf.yo
+++ b/rsyncd.conf.yo
@@ -510,6 +510,7 @@ quote(itemization(
   it() %b the number of bytes actually transferred
   it() %B the permission bits of the file (e.g. rwxrwxrwt)
   it() %c the total size of the block checksums received for the basis file (only when sending)
+  it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above).
   it() %f the filename (long form on sender; no trailing "/")
   it() %G the gid of the file (decimal) or "DEFAULT"
   it() %h the remote host name