From: Wayne Davison Date: Sun, 20 Jul 2008 05:45:05 +0000 (-0700) Subject: Added a '%C' (MD5 checksum) flag for the output/logfile formatting. X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/commitdiff_plain/886df221c1ce1660a2b6cd274b13952b482fe5bf Added a '%C' (MD5 checksum) flag for the output/logfile formatting. --- diff --git a/NEWS b/NEWS index 7be38f2b..805a6ba0 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,10 @@ Changes since 3.0.3: - Improved the keep-alive in-loop check in the generator to work properly in incremental recursion mode. + - Fixed a couple issues in the --fake-super handling of xattrs when the + destination files have root-level attributes (e.g. selinux values) that + a non-root copy can't affect. + ENHANCEMENTS: - Added the --remote-option=OPT (-M OPT) command-line option that is useful @@ -19,9 +23,22 @@ Changes since 3.0.3: control over what is output. Added an extra type of --progress output using --info=progress2. + - Added the "%C" escape to the log-output handling, which will output the + MD5 checksum of any transferred file, or all files if --checksum was + specified (when protocol 30 or above is in effect). + - Rsync will not send an -e option to the server if the user specifies the --protocol=29 option. This lets rsync3 use an overly-restrictive server. DEVELOPER RELATED: - - ... + - Added more conditional debug output. + + - Really big numbers are now output using our own big-num routine rather + than casting them to a double and using a %.0f conversion. + + - The pool_alloc library has received some minor improvements in alignment + handling. + + - The Makefile will not halt for just a timestamp change on the Makefile + or the configure files, only for actual changes in content. diff --git a/flist.c b/flist.c index 1f90403f..7c21f73e 100644 --- a/flist.c +++ b/flist.c @@ -66,6 +66,7 @@ extern int sanitize_paths; extern int munge_symlinks; extern int need_unsorted_flist; extern int output_needs_newline; +extern int sender_keeps_checksum; extern int unsort_ndx; extern struct stats stats; extern char *filesfrom_host; @@ -1209,6 +1210,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, extra_len += EXTRA_LEN; #endif + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { + file_checksum(thisname, tmp_sum, st.st_size); + if (sender_keeps_checksum) + extra_len += SUM_EXTRA_CNT * EXTRA_LEN; + } + #if EXTRA_ROUNDING > 0 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; @@ -1272,9 +1279,6 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(bp + basename_len, linkname, linkname_len); #endif - if (always_checksum && am_sender && S_ISREG(st.st_mode)) - file_checksum(thisname, tmp_sum, st.st_size); - if (am_sender) F_PATHNAME(file) = pathname; else if (!pool) @@ -1305,6 +1309,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, return NULL; } + if (sender_keeps_checksum && S_ISREG(st.st_mode)) + memcpy(F_SUM(file), tmp_sum, checksum_len); + if (unsort_ndx) F_NDX(file) = dir_count; diff --git a/log.c b/log.c index 4db96637..8f8f049f 100644 --- a/log.c +++ b/log.c @@ -31,8 +31,10 @@ extern int local_server; extern int quiet; extern int module_id; extern int msg_fd_out; +extern int checksum_len; extern int allow_8bit_chars; extern int protocol_version; +extern int always_checksum; extern int preserve_times; extern int uid_ndx; extern int gid_ndx; @@ -55,6 +57,7 @@ extern iconv_t ic_send, ic_recv; extern char curr_dir[]; extern char *module_dir; extern unsigned int module_dirlen; +extern char sender_file_sum[]; static int log_initialised; static int logfile_was_closed; @@ -633,6 +636,28 @@ static void log_formatted(enum logcode code, const char *format, const char *op, snprintf(buf2, sizeof buf2, fmt, big_num(b, 0)); n = buf2; break; + case 'C': + if (protocol_version >= 30 + && (iflags & ITEM_TRANSFER + || (always_checksum && S_ISREG(file->mode)))) { + int i, x1, x2; + const char *sum = iflags & ITEM_TRANSFER + ? sender_file_sum : F_SUM(file); + c = buf2 + checksum_len*2; + *c = '\0'; + for (i = checksum_len; --i >= 0; ) { + x1 = CVAL(sum, i); + x2 = x1 >> 4; + x1 &= 0xF; + *--c = x1 <= 9 ? x1 + '0' : x1 + 'a' - 10; + *--c = x2 <= 9 ? x2 + '0' : x2 + 'a' - 10; + } + } else { + memset(buf2, ' ', checksum_len*2); + buf2[checksum_len*2] = '\0'; + } + n = buf2; + break; case 'i': if (iflags & ITEM_DELETED) { n = "*deleting "; diff --git a/main.c b/main.c index 73440865..939004ab 100644 --- a/main.c +++ b/main.c @@ -37,6 +37,7 @@ extern int am_daemon; extern int inc_recurse; extern int blocking_io; extern int human_readable; +extern int always_checksum; extern int remove_source_files; extern int output_needs_newline; extern int need_messages_from_generator; @@ -68,6 +69,8 @@ extern int connect_timeout; extern pid_t cleanup_child_pid; extern unsigned int module_dirlen; extern struct stats stats; +extern char *stdout_format; +extern char *logfile_format; extern char *filesfrom_host; extern char *partial_dir; extern char *dest_option; @@ -85,6 +88,7 @@ int local_server = 0; int daemon_over_rsh = 0; mode_t orig_umask = 0; int batch_gen_fd = -1; +int sender_keeps_checksum = 0; /* There's probably never more than at most 2 outstanding child processes, * but set it higher, just in case. */ @@ -1013,6 +1017,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) if (am_sender) { keep_dirlinks = 0; /* Must be disabled on the sender. */ + + if (always_checksum + && (log_format_has(stdout_format, 'C') + || log_format_has(logfile_format, 'C'))) + sender_keeps_checksum = 1; + if (protocol_version >= 30) io_start_multiplex_out(); else diff --git a/match.c b/match.c index a3659fc6..0d4cfeb1 100644 --- a/match.c +++ b/match.c @@ -23,8 +23,10 @@ extern int checksum_seed; extern int append_mode; +extern int checksum_len; int updating_basis_file; +char sender_file_sum[MAX_DIGEST_LEN]; static int false_alarms; static int hash_hits; @@ -327,9 +329,6 @@ static void hash_search(int f,struct sum_struct *s, **/ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) { - char file_sum[MAX_DIGEST_LEN]; - int sum_len; - last_match = 0; false_alarms = 0; hash_hits = 0; @@ -377,18 +376,28 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len) matched(f, s, buf, len, -1); } - sum_len = sum_end(file_sum); - /* If we had a read error, send a bad checksum. */ - if (buf && buf->status != 0) - file_sum[0]++; + if (sum_end(sender_file_sum) != checksum_len) + overflow_exit("checksum_len"); /* Impossible... */ + + /* If we had a read error, send a bad checksum. We use all bits + * off as long as the checksum doesn't happen to be that, in + * which case we turn the last 0 bit into a 1. */ + if (buf && buf->status != 0) { + int i; + for (i = 0; i < checksum_len && sender_file_sum[i] == 0; i++) {} + memset(sender_file_sum, 0, checksum_len); + if (i == checksum_len) + sender_file_sum[i-1]++; + } if (DEBUG_GTE(CHKSUM, 2)) rprintf(FINFO,"sending file_sum\n"); - write_buf(f, file_sum, sum_len); + write_buf(f, sender_file_sum, checksum_len); - if (DEBUG_GTE(CHKSUM, 2)) + if (DEBUG_GTE(CHKSUM, 2)) { rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n", false_alarms, hash_hits, matches); + } total_hash_hits += hash_hits; total_false_alarms += false_alarms; diff --git a/options.c b/options.c index 739ae911..4f6b5e14 100644 --- a/options.c +++ b/options.c @@ -1901,7 +1901,8 @@ int parse_arguments(int *argc_p, const char ***argv_p) else if (log_format_has(stdout_format, 'i')) stdout_format_has_i = itemize_changes | 1; if (!log_format_has(stdout_format, 'b') - && !log_format_has(stdout_format, 'c')) + && !log_format_has(stdout_format, 'c') + && !log_format_has(stdout_format, 'C')) log_before_transfer = !am_server; } else if (itemize_changes) { stdout_format = "%i %n%L"; diff --git a/receiver.c b/receiver.c index 05ac184c..ffaf3e40 100644 --- a/receiver.c +++ b/receiver.c @@ -44,6 +44,7 @@ extern int remove_source_files; extern int append_mode; extern int sparse_files; extern int keep_partial; +extern int checksum_len; extern int checksum_seed; extern int inplace; extern int delay_updates; @@ -52,6 +53,7 @@ extern struct stats stats; extern char *tmpdir; extern char *partial_dir; extern char *basis_dir[]; +extern char sender_file_sum[]; extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct filter_list_struct daemon_filter_list; @@ -163,10 +165,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, const char *fname, int fd, OFF_T total_size) { static char file_sum1[MAX_DIGEST_LEN]; - static char file_sum2[MAX_DIGEST_LEN]; struct map_struct *mapbuf; struct sum_struct sum; - int32 len, sum_len; + int32 len; OFF_T offset = 0; OFF_T offset2; char *data; @@ -296,15 +297,16 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, exit_cleanup(RERR_FILEIO); } - sum_len = sum_end(file_sum1); + if (sum_end(file_sum1) != checksum_len) + overflow_exit("checksum_len"); /* Impossible... */ if (mapbuf) unmap_file(mapbuf); - read_buf(f_in, file_sum2, sum_len); + read_buf(f_in, sender_file_sum, checksum_len); if (DEBUG_GTE(CHKSUM, 2)) rprintf(FINFO,"got file_sum\n"); - if (fd != -1 && memcmp(file_sum1, file_sum2, sum_len) != 0) + if (fd != -1 && memcmp(file_sum1, sender_file_sum, checksum_len) != 0) return 0; return 1; } diff --git a/rsync.yo b/rsync.yo index 1b30c7a4..c34d4be7 100644 --- a/rsync.yo +++ b/rsync.yo @@ -2182,7 +2182,7 @@ by the server and defaults to the current code(time()). This option is used to set a specific checksum seed, which is useful for applications that want repeatable block and file checksums, or in the case where the user wants a more random checksum seed. -Note that setting NUM to 0 causes rsync to use the default of code(time()) +Setting NUM to 0 causes rsync to use the default of code(time()) for checksum seed. enddit() diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo index 5fe4e202..b3ebd87e 100644 --- a/rsyncd.conf.yo +++ b/rsyncd.conf.yo @@ -509,7 +509,8 @@ quote(itemization( it() %a the remote IP address it() %b the number of bytes actually transferred it() %B the permission bits of the file (e.g. rwxrwxrwt) - it() %c the checksum bytes received for this file (only when sending) + it() %c the total size of the block checksums received for the basis file (only when sending) + it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above). it() %f the filename (long form on sender; no trailing "/") it() %G the gid of the file (decimal) or "DEFAULT" it() %h the remote host name