Commit | Line | Data |
---|---|---|
77945073 WD |
1 | Added a pipe from the generator to the receiver that communicates |
2 | what basis file we used to generate the file data (if it was not | |
3 | the default name). This optimizes away the basis-file search in | |
4 | the receiver and makes future options that do more basis-file | |
5 | searching more efficient (such as the --fuzzy option and the | |
6 | support for multiple --compare-dest options). | |
7 | ||
8 | You must run "make proto" before compiling. | |
9 | ||
2aa9af10 | 10 | --- orig/generator.c 2004-10-06 00:12:16 |
84f5a99a | 11 | +++ generator.c 2004-11-03 20:12:24 |
ccfe7463 | 12 | @@ -254,14 +254,15 @@ static void generate_and_send_sums(int f |
77945073 WD |
13 | * out. It might be wrong. |
14 | */ | |
15 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
16 | - int f_out) | |
d6e18487 | 17 | + int f_out, int f_out_name) |
77945073 | 18 | { |
ccfe7463 | 19 | - int fd, f_copy; |
2aa9af10 | 20 | + int fd = -1, f_copy = -1; |
ccfe7463 | 21 | STRUCT_STAT st, partial_st; |
2aa9af10 WD |
22 | - struct file_struct *back_file; |
23 | + struct file_struct *back_file = NULL; | |
b952a177 | 24 | int statret, stat_errno; |
2aa9af10 WD |
25 | - char *fnamecmp, *partialptr, *backupptr; |
26 | + char *fnamecmp, *partialptr, *backupptr = NULL; | |
ccfe7463 | 27 | char fnamecmpbuf[MAXPATHLEN]; |
65021ef6 WD |
28 | + uchar fnamecmp_type; |
29 | ||
30 | if (list_only) | |
31 | return; | |
2aa9af10 | 32 | @@ -421,6 +422,7 @@ static void recv_generator(char *fname, |
65021ef6 WD |
33 | } |
34 | ||
35 | fnamecmp = fname; | |
e47edaf6 | 36 | + fnamecmp_type = FNAMECMP_FNAME; |
65021ef6 WD |
37 | |
38 | if (statret == -1 && compare_dest != NULL) { | |
39 | /* try the file at compare_dest instead */ | |
2aa9af10 | 40 | @@ -437,10 +439,14 @@ static void recv_generator(char *fname, |
b952a177 WD |
41 | safe_fname(fname)); |
42 | } | |
43 | fnamecmp = fnamecmpbuf; | |
84f5a99a | 44 | + fnamecmp_type = FNAMECMP_CMPDEST; |
65021ef6 | 45 | } |
b952a177 | 46 | } else |
77945073 | 47 | #endif |
65021ef6 | 48 | + { |
b952a177 | 49 | fnamecmp = fnamecmpbuf; |
84f5a99a | 50 | + fnamecmp_type = FNAMECMP_CMPDEST; |
65021ef6 | 51 | + } |
b952a177 | 52 | statret = 0; |
65021ef6 WD |
53 | } |
54 | } | |
2aa9af10 | 55 | @@ -463,11 +469,9 @@ static void recv_generator(char *fname, |
77945073 WD |
56 | if (statret == -1) { |
57 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
58 | return; | |
b952a177 | 59 | - if (stat_errno == ENOENT) { |
0bd6d59f | 60 | - write_int(f_out,i); |
2f5fa77e | 61 | - if (!dry_run && !read_batch) |
0bd6d59f WD |
62 | - write_sum_head(f_out, NULL); |
63 | - } else if (verbose > 1) { | |
b952a177 | 64 | + if (stat_errno == ENOENT) |
d24c084d | 65 | + goto notify_others; |
0bd6d59f | 66 | + if (verbose > 1) { |
b952a177 WD |
67 | rsyserr(FERROR, stat_errno, |
68 | "recv_generator: failed to stat %s", | |
0bd6d59f | 69 | full_fname(fname)); |
2aa9af10 | 70 | @@ -475,13 +479,13 @@ static void recv_generator(char *fname, |
b952a177 | 71 | return; |
77945073 WD |
72 | } |
73 | ||
74 | - if (opt_ignore_existing && fnamecmp == fname) { | |
e47edaf6 | 75 | + if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) { |
77945073 | 76 | if (verbose > 1) |
b952a177 | 77 | rprintf(FINFO, "%s exists\n", safe_fname(fname)); |
77945073 WD |
78 | return; |
79 | } | |
80 | ||
81 | - if (update_only && fnamecmp == fname | |
e47edaf6 | 82 | + if (update_only && fnamecmp_type == FNAMECMP_FNAME |
77945073 WD |
83 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { |
84 | if (verbose > 1) | |
b952a177 | 85 | rprintf(FINFO, "%s is newer\n", safe_fname(fname)); |
2aa9af10 | 86 | @@ -489,26 +493,23 @@ static void recv_generator(char *fname, |
77945073 WD |
87 | } |
88 | ||
ec8473c6 | 89 | if (skip_file(fnamecmp, file, &st)) { |
77945073 | 90 | - if (fnamecmp == fname) |
e47edaf6 | 91 | + if (fnamecmp_type == FNAMECMP_FNAME) |
77945073 WD |
92 | set_perms(fname, file, &st, PERMS_REPORT); |
93 | return; | |
94 | } | |
95 | ||
ccfe7463 | 96 | prepare_to_open: |
2f5fa77e | 97 | - if (dry_run || read_batch) { |
0bd6d59f WD |
98 | - write_int(f_out,i); |
99 | - return; | |
100 | - } | |
101 | - | |
102 | - if (whole_file > 0) { | |
103 | - write_int(f_out,i); | |
104 | - write_sum_head(f_out, NULL); | |
105 | - return; | |
b857cf41 | 106 | + if (dry_run || whole_file > 0) { |
0bd6d59f | 107 | + statret = -1; |
d24c084d | 108 | + goto notify_others; |
77945073 | 109 | } |
0bd6d59f | 110 | + if (read_batch) |
d24c084d | 111 | + goto notify_others; |
77945073 | 112 | |
ccfe7463 WD |
113 | if (partialptr) { |
114 | st = partial_st; | |
115 | fnamecmp = partialptr; | |
e47edaf6 | 116 | + fnamecmp_type = FNAMECMP_PARTIAL_DIR; |
65021ef6 WD |
117 | } |
118 | ||
ccfe7463 | 119 | /* open the file */ |
2aa9af10 | 120 | @@ -521,9 +522,8 @@ prepare_to_open: |
77945073 WD |
121 | /* pretend the file didn't exist */ |
122 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
123 | return; | |
0bd6d59f WD |
124 | - write_int(f_out,i); |
125 | - write_sum_head(f_out, NULL); | |
126 | - return; | |
127 | + statret = -1; | |
d24c084d | 128 | + goto notify_others; |
0bd6d59f WD |
129 | } |
130 | ||
ccfe7463 | 131 | if (inplace && make_backups) { |
2aa9af10 | 132 | @@ -550,10 +550,7 @@ prepare_to_open: |
ccfe7463 WD |
133 | close(fd); |
134 | return; | |
135 | } | |
2aa9af10 WD |
136 | - } else { |
137 | - backupptr = NULL; | |
138 | - back_file = NULL; | |
139 | - f_copy = -1; | |
e47edaf6 | 140 | + fnamecmp_type = FNAMECMP_BACKUP; |
2aa9af10 WD |
141 | } |
142 | ||
143 | if (verbose > 3) { | |
e47edaf6 | 144 | @@ -564,22 +561,35 @@ prepare_to_open: |
77945073 WD |
145 | if (verbose > 2) |
146 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
147 | ||
0bd6d59f | 148 | - write_int(f_out,i); |
ccfe7463 WD |
149 | - generate_and_send_sums(fd, st.st_size, f_out, f_copy); |
150 | - | |
151 | - if (f_copy >= 0) { | |
152 | - close(f_copy); | |
153 | - set_perms(backupptr, back_file, NULL, 0); | |
a6587818 WD |
154 | - if (verbose > 1) |
155 | - rprintf(FINFO, "backed up %s to %s\n", fname, backupptr); | |
ccfe7463 | 156 | - free(back_file); |
e47edaf6 | 157 | - } |
d24c084d | 158 | +notify_others: |
ccfe7463 | 159 | + write_int(f_out, i); |
e47edaf6 WD |
160 | + if (f_out_name >= 0) |
161 | + write_byte(f_out_name, fnamecmp_type); | |
ccfe7463 | 162 | + |
65021ef6 WD |
163 | + if (dry_run || read_batch) |
164 | + return; | |
165 | + | |
0bd6d59f | 166 | + if (statret == 0) { |
ccfe7463 | 167 | + generate_and_send_sums(fd, st.st_size, f_out, f_copy); |
ccfe7463 WD |
168 | + |
169 | + if (f_copy >= 0) { | |
170 | + close(f_copy); | |
171 | + set_perms(backupptr, back_file, NULL, 0); | |
a6587818 WD |
172 | + if (verbose > 1) { |
173 | + rprintf(FINFO, "backed up %s to %s\n", | |
174 | + fname, backupptr); | |
175 | + } | |
ccfe7463 WD |
176 | + free(back_file); |
177 | + } | |
e47edaf6 WD |
178 | |
179 | - close(fd); | |
a6587818 | 180 | + close(fd); |
d6e18487 | 181 | + } else |
0bd6d59f | 182 | + write_sum_head(f_out, NULL); |
77945073 WD |
183 | } |
184 | ||
185 | ||
9be39c35 WD |
186 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
187 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
d6e18487 | 188 | + int f_out_name) |
77945073 WD |
189 | { |
190 | int i; | |
8cec1ead | 191 | int phase = 0; |
e47edaf6 | 192 | @@ -620,7 +630,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
193 | } |
194 | ||
195 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 196 | - file, i, f_out); |
d6e18487 | 197 | + file, i, f_out, f_out_name); |
77945073 WD |
198 | } |
199 | ||
200 | phase++; | |
e47edaf6 | 201 | @@ -637,7 +647,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
202 | while ((i = get_redo_num()) != -1) { |
203 | struct file_struct *file = flist->files[i]; | |
204 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 205 | - file, i, f_out); |
d6e18487 | 206 | + file, i, f_out, f_out_name); |
77945073 WD |
207 | } |
208 | ||
209 | phase++; | |
e47edaf6 | 210 | @@ -656,7 +666,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
211 | if (!file->basename || !S_ISDIR(file->mode)) |
212 | continue; | |
213 | recv_generator(local_name ? local_name : f_name(file), | |
214 | - file, i, -1); | |
215 | + file, i, -1, -1); | |
216 | } | |
217 | ||
218 | if (verbose > 2) | |
e47edaf6 WD |
219 | --- orig/io.c 2004-08-02 02:44:26 |
220 | +++ io.c 2004-11-03 19:49:36 | |
221 | @@ -857,6 +857,11 @@ static void writefd_unbuffered(int fd,ch | |
222 | if (msg_fd_in > maxfd) | |
223 | maxfd = msg_fd_in; | |
224 | } | |
225 | + if (fd != sock_f_out && iobuf_out_cnt && no_flush == 1) { | |
226 | + FD_SET(sock_f_out, &w_fds); | |
227 | + if (sock_f_out > maxfd) | |
228 | + maxfd = sock_f_out; | |
229 | + } | |
230 | ||
231 | tv.tv_sec = select_timeout; | |
232 | tv.tv_usec = 0; | |
233 | @@ -875,8 +880,14 @@ static void writefd_unbuffered(int fd,ch | |
234 | if (msg_fd_in >= 0 && FD_ISSET(msg_fd_in, &r_fds)) | |
235 | read_msg_fd(); | |
236 | ||
237 | - if (!FD_ISSET(fd, &w_fds)) | |
238 | + if (!FD_ISSET(fd, &w_fds)) { | |
239 | + if (fd != sock_f_out && iobuf_out_cnt) { | |
240 | + no_flush--; | |
241 | + io_flush(NORMAL_FLUSH); | |
242 | + no_flush++; | |
243 | + } | |
244 | continue; | |
245 | + } | |
246 | ||
247 | n = len - total; | |
248 | if (bwlimit && n > bwlimit_writemax) | |
2aa9af10 | 249 | --- orig/main.c 2004-10-08 20:16:26 |
e47edaf6 | 250 | +++ main.c 2004-11-03 16:07:53 |
d5753a22 | 251 | @@ -59,6 +59,7 @@ extern int filesfrom_fd; |
0bd6d59f WD |
252 | extern pid_t cleanup_child_pid; |
253 | extern char *files_from; | |
254 | extern char *remote_filesfrom_file; | |
255 | +extern char *compare_dest; | |
256 | extern char *rsync_path; | |
257 | extern char *shell_cmd; | |
258 | extern char *batch_name; | |
f635ed27 | 259 | @@ -461,7 +462,8 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
260 | { |
261 | int pid; | |
8cec1ead | 262 | int status = 0; |
77945073 | 263 | - int error_pipe[2]; |
77945073 | 264 | + int error_pipe[2], name_pipe[2]; |
d6e18487 | 265 | + BOOL need_name_pipe = compare_dest && !dry_run; |
77945073 | 266 | |
d5753a22 WD |
267 | /* The receiving side mustn't obey this, or an existing symlink that |
268 | * points to an identical file won't be replaced by the referent. */ | |
2aa9af10 | 269 | @@ -476,7 +478,8 @@ static int do_recv(int f_in,int f_out,st |
d24c084d | 270 | delete_files(flist); |
77945073 WD |
271 | } |
272 | ||
273 | - if (fd_pair(error_pipe) < 0) { | |
0bd6d59f WD |
274 | + if (fd_pair(error_pipe) < 0 |
275 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
2aa9af10 | 276 | rsyserr(FERROR, errno, "pipe failed in do_recv"); |
77945073 WD |
277 | exit_cleanup(RERR_SOCKETIO); |
278 | } | |
f635ed27 | 279 | @@ -485,6 +488,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 280 | |
8cec1ead | 281 | if ((pid = do_fork()) == 0) { |
77945073 | 282 | close(error_pipe[0]); |
0bd6d59f WD |
283 | + if (need_name_pipe) { |
284 | + close(name_pipe[1]); | |
285 | + set_blocking(name_pipe[0]); | |
286 | + } else | |
287 | + name_pipe[0] = -1; | |
77945073 WD |
288 | if (f_in != f_out) |
289 | close(f_out); | |
77945073 | 290 | |
f635ed27 | 291 | @@ -494,7 +502,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
292 | /* set place to send errors */ |
293 | set_msg_fd_out(error_pipe[1]); | |
294 | ||
295 | - recv_files(f_in,flist,local_name); | |
296 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
297 | io_flush(FULL_FLUSH); | |
298 | report(f_in); | |
299 | ||
f635ed27 | 300 | @@ -513,6 +521,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 301 | stop_write_batch(); |
77945073 WD |
302 | |
303 | close(error_pipe[1]); | |
0bd6d59f WD |
304 | + if (need_name_pipe) { |
305 | + close(name_pipe[0]); | |
306 | + set_nonblocking(name_pipe[1]); | |
307 | + } else | |
308 | + name_pipe[1] = -1; | |
77945073 WD |
309 | if (f_in != f_out) |
310 | close(f_in); | |
77945073 | 311 | |
f635ed27 | 312 | @@ -520,7 +533,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
313 | |
314 | set_msg_fd_in(error_pipe[0]); | |
315 | ||
316 | - generate_files(f_out, flist, local_name); | |
317 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
318 | ||
319 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
320 | report(-1); | |
e47edaf6 | 321 | --- orig/receiver.c 2004-11-03 11:56:04 |
84f5a99a | 322 | +++ receiver.c 2004-11-03 20:16:38 |
e47edaf6 WD |
323 | @@ -30,6 +30,7 @@ extern int dry_run; |
324 | extern int read_batch; | |
325 | extern int batch_gen_fd; | |
326 | extern int am_server; | |
327 | +extern int protocol_version; | |
328 | extern int relative_paths; | |
329 | extern int keep_dirlinks; | |
330 | extern int preserve_hard_links; | |
331 | @@ -329,7 +330,8 @@ static void discard_receive_data(int f_i | |
77945073 WD |
332 | * main routine for receiver process. |
333 | * | |
334 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 335 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 336 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
d6e18487 | 337 | + int f_in_name) |
77945073 | 338 | { |
2f5fa77e | 339 | int next_gen_i = -1; |
77945073 | 340 | int fd1,fd2; |
e47edaf6 | 341 | @@ -358,8 +360,13 @@ int recv_files(int f_in, struct file_lis |
0bd6d59f WD |
342 | i = read_int(f_in); |
343 | if (i == -1) { | |
2f5fa77e WD |
344 | if (read_batch) { |
345 | - if (next_gen_i != flist->count) | |
346 | - while (read_int(batch_gen_fd) != -1) {} | |
65021ef6 WD |
347 | + if (next_gen_i != flist->count) { |
348 | + do { | |
349 | + if (f_in_name >= 0 | |
350 | + && next_gen_i >= 0) | |
351 | + read_byte(f_in_name); | |
352 | + } while (read_int(batch_gen_fd) != -1); | |
0bd6d59f | 353 | + } |
2f5fa77e WD |
354 | next_gen_i = -1; |
355 | } | |
5823d322 | 356 | |
e47edaf6 | 357 | @@ -407,6 +414,8 @@ int recv_files(int f_in, struct file_lis |
77945073 | 358 | |
2f5fa77e WD |
359 | if (read_batch) { |
360 | while (i > next_gen_i) { | |
65021ef6 WD |
361 | + if (f_in_name >= 0 && next_gen_i >= 0) |
362 | + read_byte(f_in_name); | |
2f5fa77e WD |
363 | next_gen_i = read_int(batch_gen_fd); |
364 | if (next_gen_i == -1) | |
365 | next_gen_i = flist->count; | |
e47edaf6 | 366 | @@ -417,6 +426,7 @@ int recv_files(int f_in, struct file_lis |
2f5fa77e WD |
367 | discard_receive_data(f_in, file->length); |
368 | continue; | |
369 | } | |
0bd6d59f | 370 | + next_gen_i = -1; |
2f5fa77e WD |
371 | } |
372 | ||
5823d322 | 373 | if (server_exclude_list.head |
84f5a99a | 374 | @@ -426,35 +436,32 @@ int recv_files(int f_in, struct file_lis |
65021ef6 | 375 | exit_cleanup(RERR_PROTOCOL); |
b952a177 WD |
376 | } |
377 | ||
378 | - if (partial_dir) { | |
379 | - if ((partialptr = partial_dir_fname(fname)) != NULL) | |
380 | - fnamecmp = partialptr; | |
381 | - else | |
b952a177 | 382 | + partialptr = partial_dir ? partial_dir_fname(fname) : fname; |
65021ef6 WD |
383 | + |
384 | + if (f_in_name >= 0) { | |
385 | + switch (read_byte(f_in_name)) { | |
e47edaf6 | 386 | + case FNAMECMP_FNAME: |
65021ef6 WD |
387 | fnamecmp = fname; |
388 | + break; | |
e47edaf6 | 389 | + case FNAMECMP_PARTIAL_DIR: |
65021ef6 WD |
390 | + fnamecmp = partialptr ? partialptr : fname; |
391 | + break; | |
e47edaf6 | 392 | + case FNAMECMP_BACKUP: |
ccfe7463 WD |
393 | + fnamecmp = get_backup_name(fname); |
394 | + break; | |
84f5a99a | 395 | + case FNAMECMP_CMPDEST: |
65021ef6 WD |
396 | + default: |
397 | + pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
398 | + compare_dest, fname); | |
399 | + fnamecmp = fnamecmpbuf; | |
400 | + break; | |
401 | + } | |
402 | } else | |
403 | - fnamecmp = partialptr = fname; | |
ccfe7463 WD |
404 | - |
405 | - if (inplace && make_backups) { | |
406 | - if (!(fnamecmp = get_backup_name(fname))) | |
407 | - fnamecmp = partialptr; | |
408 | - } | |
65021ef6 | 409 | + fnamecmp = fname; |
b952a177 | 410 | |
77945073 WD |
411 | /* open the file */ |
412 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
413 | ||
65021ef6 WD |
414 | - if (fd1 == -1 && fnamecmp != fname) { |
415 | - fnamecmp = fname; | |
416 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
417 | - } | |
418 | - | |
77945073 WD |
419 | - if (fd1 == -1 && compare_dest != NULL) { |
420 | - /* try the file at compare_dest instead */ | |
421 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
422 | - compare_dest, fname); | |
423 | - fnamecmp = fnamecmpbuf; | |
424 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
425 | - } | |
426 | - | |
427 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
428 | rsyserr(FERROR, errno, "fstat %s failed", | |
429 | full_fname(fnamecmp)); | |
2aa9af10 | 430 | --- orig/rsync.h 2004-10-09 03:21:56 |
84f5a99a WD |
431 | +++ rsync.h 2004-11-03 20:11:34 |
432 | @@ -119,6 +119,11 @@ | |
65021ef6 WD |
433 | #define PDIR_CREATE 1 |
434 | #define PDIR_DELETE 0 | |
435 | ||
84f5a99a | 436 | +#define FNAMECMP_CMPDEST 0x00 |
e47edaf6 WD |
437 | +#define FNAMECMP_FNAME 0x80 |
438 | +#define FNAMECMP_PARTIAL_DIR 0x81 | |
439 | +#define FNAMECMP_BACKUP 0x82 | |
65021ef6 WD |
440 | + |
441 | ||
442 | /* Log-message categories. FLOG is only used on the daemon side to | |
443 | * output messages to the log file. */ |