This patch changes the way the --checksum option works by having the
receiving side perform a checksum-read of every file in the file list
-(if the sizes are equal) as the list is received, marking non-matching
+as the list is received (if the sizes are equal), marking non-matching
items with a flag. The idea is that the checksum pass on the sender and
the receiver can then happen in parallel instead of having the reciever
to its checksum pass during its normal find-the-different-files pass.
-I have benchmarked this a little, and it appears to slow things down.
+I have benchmarked this a little, and it appears to slow things down
+for a local copy, so the old algorithm is used for local copies.
---- orig/flist.c 2005-11-01 20:09:55
-+++ flist.c 2005-09-16 16:41:25
-@@ -36,6 +36,7 @@ extern int am_daemon;
- extern int am_sender;
+To use this patch, run these commands for a successful build:
+
+ patch -p1 <patches/early-checksum.diff
+ ./configure (optional if already run)
+ make
+
+--- old/flist.c
++++ new/flist.c
+@@ -34,6 +34,7 @@ extern int am_generator;
+ extern int inc_recurse;
extern int do_progress;
extern int always_checksum;
+extern int pre_checksum;
extern int module_id;
extern int ignore_errors;
extern int numeric_ids;
-@@ -705,6 +706,16 @@ static struct file_struct *receive_file_
- sum = empty_sum;
- }
- read_buf(f, sum, slen);
-+ if (pre_checksum) {
-+ char sum2[MD4_SUM_LENGTH];
+@@ -949,6 +950,15 @@ static struct file_struct *recv_file_ent
+ memcpy(bp, F_SUM(first), checksum_len);
+ } else
+ read_buf(f, bp, checksum_len);
++ if (pre_checksum && bp != tmp_sum) {
+ STRUCT_STAT st;
-+ char *fname = f_name(file);
++ char *fname = f_name(file, NULL);
+ if (stat(fname, &st) == 0 && st.st_size == file_length) {
-+ file_checksum(fname, sum2, st.st_size);
-+ if (memcmp(sum, sum2, slen) != 0)
++ file_checksum(fname, tmp_sum, st.st_size);
++ if (memcmp(bp, tmp_sum, checksum_len) != 0)
+ file->flags |= FLAG_SUM_DIFFERS;
+ }
+ }
}
- if (!preserve_perms) {
---- orig/generator.c 2005-10-30 22:30:28
-+++ generator.c 2005-09-16 16:41:25
-@@ -69,6 +69,7 @@ extern int ignore_timeout;
- extern int protocol_version;
+ #ifdef SUPPORT_ACLS
+--- old/generator.c
++++ new/generator.c
+@@ -73,6 +73,7 @@ extern int protocol_version;
+ extern int file_total;
extern int fuzzy_basis;
extern int always_checksum;
+extern int pre_checksum;
+ extern int checksum_len;
extern char *partial_dir;
extern char *basis_dir[];
- extern int compare_dest;
-@@ -359,7 +360,8 @@ void itemize(struct file_struct *file, i
+@@ -617,7 +618,8 @@ void itemize(const char *fnamecmp, struc
/* Perform our quick-check heuristic for determining if a file is unchanged. */
--static int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
-+static int unchanged_file(char *fn, int fnamecmp_type, struct file_struct *file,
-+ STRUCT_STAT *st)
+-int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
++int unchanged_file(char *fn, int fnamecmp_type, struct file_struct *file,
++ STRUCT_STAT *st)
{
- if (st->st_size != file->length)
+ if (st->st_size != F_LENGTH(file))
return 0;
-@@ -368,6 +370,8 @@ static int unchanged_file(char *fn, stru
+@@ -626,6 +628,8 @@ int unchanged_file(char *fn, struct file
of the file time to determine whether to sync */
- if (always_checksum && S_ISREG(st->st_mode)) {
- char sum[MD4_SUM_LENGTH];
+ if (always_checksum > 0 && S_ISREG(st->st_mode)) {
+ char sum[MAX_DIGEST_LEN];
+ if (pre_checksum && fnamecmp_type == FNAMECMP_FNAME)
+ return !(file->flags & FLAG_SUM_DIFFERS);
file_checksum(fn, sum, st->st_size);
- return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
- : MD4_SUM_LENGTH) == 0;
-@@ -923,7 +927,7 @@ static void recv_generator(char *fname,
- match_level = 1;
- /* FALL THROUGH */
- case 1:
-- if (!unchanged_file(fnamecmpbuf, file, &st))
-+ if (!unchanged_file(fnamecmpbuf, 0, file, &st))
- continue;
- best_match = i;
- match_level = 2;
-@@ -1051,7 +1055,7 @@ static void recv_generator(char *fname,
+ return memcmp(sum, F_SUM(file), checksum_len) == 0;
+ }
+@@ -846,7 +850,7 @@ static int try_dests_reg(struct file_str
+ match_level = 1;
+ /* FALL THROUGH */
+ case 1:
+- if (!unchanged_file(cmpbuf, file, &sxp->st))
++ if (!unchanged_file(cmpbuf, 0, file, &sxp->st))
+ continue;
+ best_match = j;
+ match_level = 2;
+@@ -1638,7 +1642,7 @@ static void recv_generator(char *fname,
;
else if (fnamecmp_type == FNAMECMP_FUZZY)
;
-- else if (unchanged_file(fnamecmp, file, &st)) {
-+ else if (unchanged_file(fnamecmp, fnamecmp_type, file, &st)) {
- if (fnamecmp_type == FNAMECMP_FNAME) {
- if (itemizing) {
- itemize(file, ndx, real_ret, &real_st,
---- orig/main.c 2005-11-02 07:22:12
-+++ main.c 2005-09-16 16:41:25
-@@ -45,6 +45,7 @@ extern int copy_links;
+- else if (unchanged_file(fnamecmp, file, &sx.st)) {
++ else if (unchanged_file(fnamecmp, fnamecmp_type, file, &sx.st)) {
+ if (partialptr) {
+ do_unlink(partialptr);
+ handle_partial_dir(partialptr, PDIR_DELETE);
+--- old/hlink.c
++++ new/hlink.c
+@@ -353,7 +353,7 @@ int hard_link_check(struct file_struct *
+ }
+ break;
+ }
+- if (!unchanged_file(cmpbuf, file, &alt_sx.st))
++ if (!unchanged_file(cmpbuf, 0, file, &alt_sx.st))
+ continue;
+ statret = 1;
+ if (unchanged_attrs(cmpbuf, file, &alt_sx))
+--- old/main.c
++++ new/main.c
+@@ -46,6 +46,7 @@ extern int module_id;
+ extern int copy_links;
+ extern int copy_dirlinks;
extern int keep_dirlinks;
++extern int always_checksum;
extern int preserve_hard_links;
extern int protocol_version;
-+extern int always_checksum;
- extern int recurse;
- extern int relative_paths;
- extern int rsync_port;
-@@ -60,8 +61,10 @@ extern char *filesfrom_host;
- extern char *rsync_path;
- extern char *shell_cmd;
- extern char *batch_name;
-+extern char curr_dir[MAXPATHLEN];
+ extern int file_total;
+@@ -78,6 +79,9 @@ extern struct filter_list_struct server_
+ extern iconv_t ic_send;
+ #endif
- int local_server = 0;
++extern char curr_dir[MAXPATHLEN];
++
+int pre_checksum = 0;
- struct file_list *the_file_list;
-
- /* There's probably never more than at most 2 outstanding child processes,
-@@ -633,6 +636,7 @@ static void do_server_recv(int f_in, int
+ int local_server = 0;
+ int daemon_over_rsh = 0;
+ mode_t orig_umask = 0;
+@@ -850,6 +854,7 @@ static void do_server_recv(int f_in, int
struct file_list *flist;
char *local_name = NULL;
char *dir = NULL;
int save_verbose = verbose;
if (filesfrom_fd >= 0) {
-@@ -677,6 +681,10 @@ static void do_server_recv(int f_in, int
+@@ -896,6 +901,10 @@ static void do_server_recv(int f_in, int
filesfrom_fd = -1;
}
+ strlcpy(olddir, curr_dir, sizeof olddir);
-+ if (always_checksum && argc > 0)
-+ pre_checksum = push_dir(argv[0]);
++ if (always_checksum && !local_server && argc > 0)
++ pre_checksum = push_dir(argv[0], 0);
+
flist = recv_file_list(f_in);
- verbose = save_verbose;
if (!flist) {
-@@ -685,6 +693,9 @@ static void do_server_recv(int f_in, int
- }
- the_file_list = flist;
+ rprintf(FERROR,"server_recv: recv_file_list error\n");
+@@ -905,6 +914,9 @@ static void do_server_recv(int f_in, int
+ recv_additional_file_list(f_in);
+ verbose = save_verbose;
+ if (pre_checksum)
+ pop_dir(olddir);
if (argc > 0)
local_name = get_local_name(flist,argv[0]);
-@@ -733,6 +744,7 @@ int client_run(int f_in, int f_out, pid_
+@@ -981,6 +993,7 @@ int client_run(int f_in, int f_out, pid_
{
struct file_list *flist = NULL;
int exit_code = 0, exit_code2 = 0;
char *local_name = NULL;
cleanup_child_pid = pid;
-@@ -804,11 +816,18 @@ int client_run(int f_in, int f_out, pid_
+@@ -1058,12 +1071,19 @@ int client_run(int f_in, int f_out, pid_
filesfrom_fd = -1;
}
+ strlcpy(olddir, curr_dir, sizeof olddir);
-+ if (always_checksum)
-+ pre_checksum = push_dir(argv[0]);
++ if (always_checksum && !local_server)
++ pre_checksum = push_dir(argv[0], 0);
+
if (write_batch && !am_server)
start_write_batch(f_in);
flist = recv_file_list(f_in);
- the_file_list = flist;
+ if (inc_recurse && file_total == 1)
+ recv_additional_file_list(f_in);
+ if (pre_checksum)
+ pop_dir(olddir);
+
- if (flist && flist->count > 0) {
+ if (flist && flist->used > 0) {
local_name = get_local_name(flist, argv[0]);
---- orig/rsync.h 2005-10-14 18:45:50
-+++ rsync.h 2005-09-16 16:41:26
-@@ -64,6 +64,7 @@
- #define FLAG_DEL_HERE (1<<3) /* receiver/generator */
- #define FLAG_SENT (1<<3) /* sender */
- #define FLAG_HLINK_TOL (1<<4) /* receiver/generator */
-+#define FLAG_SUM_DIFFERS (1<<5) /* receiver/generator */
+--- old/rsync.h
++++ new/rsync.h
+@@ -73,6 +73,7 @@
+ #define FLAG_HLINK_DONE (1<<8) /* receiver/generator */
+ #define FLAG_LENGTH64 (1<<9) /* sender/receiver/generator */
+ #define FLAG_SKIP_GROUP (1<<10) /* receiver/generator */
++#define FLAG_SUM_DIFFERS (1<<11)/* receiver/generator */
+
+ /* These flags are passed to functions but not stored. */
- /* update this if you make incompatible changes */
- #define PROTOCOL_VERSION 29
Matt McCutchen's Web Site / rsync / rsync-patches.git / blobdiff