1 This patch changes the way the --checksum option works by having the
2 receiving side perform a checksum-read of every file in the file list
3 as the list is received (if the sizes are equal), marking non-matching
4 items with a flag. The idea is that the checksum pass on the sender and
5 the receiver can then happen in parallel instead of having the reciever
6 to its checksum pass during its normal find-the-different-files pass.
8 I have benchmarked this a little, and it appears to slow things down
9 for a local copy, so the old algorithm is used for local copies.
11 To use this patch, run these commands for a successful build:
13 patch -p1 <patches/early-checksum.diff
14 ./configure (optional if already run)
19 @@ -33,6 +33,7 @@ extern int am_generator;
20 extern int inc_recurse;
21 extern int do_progress;
22 extern int always_checksum;
23 +extern int pre_checksum;
25 extern int ignore_errors;
26 extern int numeric_ids;
27 @@ -955,6 +956,15 @@ static struct file_struct *recv_file_ent
28 memcpy(bp, F_SUM(first), checksum_len);
30 read_buf(f, bp, checksum_len);
31 + if (pre_checksum && bp != tmp_sum) {
33 + char *fname = f_name(file, NULL);
34 + if (stat(fname, &st) == 0 && st.st_size == file_length) {
35 + file_checksum(fname, tmp_sum, st.st_size);
36 + if (memcmp(bp, tmp_sum, checksum_len) != 0)
37 + file->flags |= FLAG_SUM_DIFFERS;
45 @@ -73,6 +73,7 @@ extern int protocol_version;
46 extern int file_total;
47 extern int fuzzy_basis;
48 extern int always_checksum;
49 +extern int pre_checksum;
50 extern int checksum_len;
51 extern char *partial_dir;
52 extern char *basis_dir[];
53 @@ -618,7 +619,8 @@ void itemize(const char *fnamecmp, struc
56 /* Perform our quick-check heuristic for determining if a file is unchanged. */
57 -int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
58 +int unchanged_file(char *fn, int fnamecmp_type, struct file_struct *file,
61 if (st->st_size != F_LENGTH(file))
63 @@ -627,6 +629,8 @@ int unchanged_file(char *fn, struct file
64 of the file time to determine whether to sync */
65 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
66 char sum[MAX_DIGEST_LEN];
67 + if (pre_checksum && fnamecmp_type == FNAMECMP_FNAME)
68 + return !(file->flags & FLAG_SUM_DIFFERS);
69 file_checksum(fn, sum, st->st_size);
70 return memcmp(sum, F_SUM(file), checksum_len) == 0;
72 @@ -847,7 +851,7 @@ static int try_dests_reg(struct file_str
76 - if (!unchanged_file(cmpbuf, file, &sxp->st))
77 + if (!unchanged_file(cmpbuf, 0, file, &sxp->st))
81 @@ -1575,7 +1579,7 @@ static void recv_generator(char *fname,
83 else if (fnamecmp_type == FNAMECMP_FUZZY)
85 - else if (unchanged_file(fnamecmp, file, &sx.st)) {
86 + else if (unchanged_file(fnamecmp, fnamecmp_type, file, &sx.st)) {
88 do_unlink(partialptr);
89 handle_partial_dir(partialptr, PDIR_DELETE);
92 @@ -392,7 +392,7 @@ int hard_link_check(struct file_struct *
96 - if (!unchanged_file(cmpbuf, file, &alt_sx.st))
97 + if (!unchanged_file(cmpbuf, 0, file, &alt_sx.st))
100 if (unchanged_attrs(cmpbuf, file, &alt_sx))
103 @@ -45,6 +45,7 @@ extern int module_id;
104 extern int copy_links;
105 extern int copy_dirlinks;
106 extern int keep_dirlinks;
107 +extern int always_checksum;
108 extern int preserve_hard_links;
109 extern int protocol_version;
110 extern int file_total;
111 @@ -72,6 +73,9 @@ extern char *password_file;
112 extern char curr_dir[MAXPATHLEN];
113 extern struct filter_list_struct server_filter_list;
115 +extern char curr_dir[MAXPATHLEN];
117 +int pre_checksum = 0;
118 int local_server = 0;
119 int new_root_dir = 0;
120 mode_t orig_umask = 0;
121 @@ -792,6 +796,7 @@ static void do_server_recv(int f_in, int
122 struct file_list *flist;
123 char *local_name = NULL;
125 + char olddir[sizeof curr_dir];
126 int save_verbose = verbose;
128 if (filesfrom_fd >= 0) {
129 @@ -838,6 +843,10 @@ static void do_server_recv(int f_in, int
133 + strlcpy(olddir, curr_dir, sizeof olddir);
134 + if (always_checksum && !local_server && argc > 0)
135 + pre_checksum = push_dir(argv[0], 0);
137 flist = recv_file_list(f_in);
139 rprintf(FERROR,"server_recv: recv_file_list error\n");
140 @@ -847,6 +856,9 @@ static void do_server_recv(int f_in, int
141 recv_additional_file_list(f_in);
142 verbose = save_verbose;
148 local_name = get_local_name(flist,argv[0]);
150 @@ -926,6 +938,7 @@ int client_run(int f_in, int f_out, pid_
152 struct file_list *flist = NULL;
153 int exit_code = 0, exit_code2 = 0;
154 + char olddir[sizeof curr_dir];
155 char *local_name = NULL;
157 cleanup_child_pid = pid;
158 @@ -1006,12 +1019,19 @@ int client_run(int f_in, int f_out, pid_
162 + strlcpy(olddir, curr_dir, sizeof olddir);
163 + if (always_checksum && !local_server)
164 + pre_checksum = push_dir(argv[0], 0);
166 if (write_batch && !am_server)
167 start_write_batch(f_in);
168 flist = recv_file_list(f_in);
169 if (inc_recurse && file_total == 1)
170 recv_additional_file_list(f_in);
175 if (flist && flist->used > 0) {
176 local_name = get_local_name(flist, argv[0]);
181 #define FLAG_HLINK_DONE (1<<8) /* receiver/generator */
182 #define FLAG_LENGTH64 (1<<9) /* sender/receiver/generator */
183 #define FLAG_SKIP_GROUP (1<<10) /* receiver/generator */
184 +#define FLAG_SUM_DIFFERS (1<<11)/* receiver/generator */
186 /* These flags are passed to functions but not stored. */