Commit | Line | Data |
---|---|---|
77945073 WD |
1 | Added a pipe from the generator to the receiver that communicates |
2 | what basis file we used to generate the file data (if it was not | |
3 | the default name). This optimizes away the basis-file search in | |
4 | the receiver and makes future options that do more basis-file | |
5 | searching more efficient (such as the --fuzzy option and the | |
6 | support for multiple --compare-dest options). | |
7 | ||
8 | You must run "make proto" before compiling. | |
9 | ||
d6e18487 | 10 | --- orig/generator.c 2004-07-23 17:16:12 |
2f5fa77e | 11 | +++ generator.c 2004-07-22 00:05:38 |
dc3ae351 | 12 | @@ -258,9 +258,9 @@ static void generate_and_send_sums(int f |
77945073 WD |
13 | * out. It might be wrong. |
14 | */ | |
15 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
16 | - int f_out) | |
d6e18487 | 17 | + int f_out, int f_out_name) |
77945073 | 18 | { |
0bd6d59f WD |
19 | - int fd; |
20 | + int fd = -1; | |
77945073 | 21 | STRUCT_STAT st; |
0bd6d59f WD |
22 | int statret; |
23 | char *fnamecmp; | |
d6e18487 | 24 | @@ -420,8 +420,10 @@ static void recv_generator(char *fname, |
77945073 WD |
25 | statret = link_stat(fnamecmpbuf, &st, 0); |
26 | if (!S_ISREG(st.st_mode)) | |
27 | statret = -1; | |
28 | - if (statret == -1) | |
0bd6d59f | 29 | + if (statret == -1) { |
77945073 WD |
30 | errno = saveerrno; |
31 | + *fnamecmpbuf = '\0'; | |
32 | + } | |
33 | #if HAVE_LINK | |
34 | else if (link_dest && !dry_run) { | |
35 | if (do_link(fnamecmpbuf, fname) != 0) { | |
d6e18487 | 36 | @@ -429,22 +431,22 @@ static void recv_generator(char *fname, |
77945073 WD |
37 | rsyserr(FINFO, errno, "link %s => %s", |
38 | fnamecmpbuf, fname); | |
39 | } | |
40 | - } | |
41 | - fnamecmp = fnamecmpbuf; | |
42 | + fnamecmp = fnamecmpbuf; | |
43 | + } else | |
44 | + *fnamecmpbuf = '\0'; | |
45 | } | |
46 | #endif | |
47 | else | |
48 | fnamecmp = fnamecmpbuf; | |
49 | - } | |
50 | + } else | |
51 | + *fnamecmpbuf = '\0'; | |
52 | ||
53 | if (statret == -1) { | |
54 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
55 | return; | |
0bd6d59f WD |
56 | - if (errno == ENOENT) { |
57 | - write_int(f_out,i); | |
2f5fa77e | 58 | - if (!dry_run && !read_batch) |
0bd6d59f WD |
59 | - write_sum_head(f_out, NULL); |
60 | - } else if (verbose > 1) { | |
61 | + if (errno == ENOENT) | |
d24c084d | 62 | + goto notify_others; |
0bd6d59f WD |
63 | + if (verbose > 1) { |
64 | rsyserr(FERROR, errno, | |
65 | "recv_generator: failed to open %s", | |
66 | full_fname(fname)); | |
d6e18487 | 67 | @@ -453,26 +455,23 @@ static void recv_generator(char *fname, |
0bd6d59f WD |
68 | } |
69 | ||
70 | if (!S_ISREG(st.st_mode)) { | |
71 | - if (delete_file(fname) != 0) { | |
72 | + if (delete_file(fname) != 0) | |
73 | return; | |
74 | - } | |
75 | ||
77945073 WD |
76 | /* now pretend the file didn't exist */ |
77 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
78 | return; | |
0bd6d59f | 79 | - write_int(f_out,i); |
2f5fa77e | 80 | - if (!dry_run && !read_batch) |
0bd6d59f WD |
81 | - write_sum_head(f_out, NULL); |
82 | - return; | |
83 | + statret = -1; | |
d24c084d | 84 | + goto notify_others; |
77945073 WD |
85 | } |
86 | ||
87 | - if (opt_ignore_existing && fnamecmp == fname) { | |
88 | + if (opt_ignore_existing && !*fnamecmpbuf) { | |
89 | if (verbose > 1) | |
90 | rprintf(FINFO,"%s exists\n",fname); | |
91 | return; | |
92 | } | |
93 | ||
94 | - if (update_only && fnamecmp == fname | |
95 | + if (update_only && !*fnamecmpbuf | |
96 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { | |
97 | if (verbose > 1) | |
98 | rprintf(FINFO,"%s is newer\n",fname); | |
d6e18487 | 99 | @@ -480,21 +479,17 @@ static void recv_generator(char *fname, |
77945073 WD |
100 | } |
101 | ||
102 | if (skip_file(fname, file, &st)) { | |
103 | - if (fnamecmp == fname) | |
104 | + if (!*fnamecmpbuf) | |
105 | set_perms(fname, file, &st, PERMS_REPORT); | |
106 | return; | |
107 | } | |
108 | ||
2f5fa77e | 109 | - if (dry_run || read_batch) { |
0bd6d59f WD |
110 | - write_int(f_out,i); |
111 | - return; | |
112 | - } | |
113 | - | |
114 | - if (whole_file > 0) { | |
115 | - write_int(f_out,i); | |
116 | - write_sum_head(f_out, NULL); | |
117 | - return; | |
118 | + if (dry_run || whole_file) { | |
119 | + statret = -1; | |
d24c084d | 120 | + goto notify_others; |
77945073 | 121 | } |
0bd6d59f | 122 | + if (read_batch) |
d24c084d | 123 | + goto notify_others; |
77945073 | 124 | |
0bd6d59f WD |
125 | /* open the file */ |
126 | fd = do_open(fnamecmp, O_RDONLY, 0); | |
d6e18487 | 127 | @@ -505,9 +500,8 @@ static void recv_generator(char *fname, |
77945073 WD |
128 | /* pretend the file didn't exist */ |
129 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
130 | return; | |
0bd6d59f WD |
131 | - write_int(f_out,i); |
132 | - write_sum_head(f_out, NULL); | |
133 | - return; | |
134 | + statret = -1; | |
d24c084d | 135 | + goto notify_others; |
0bd6d59f WD |
136 | } |
137 | ||
0bd6d59f | 138 | if (verbose > 3) { |
d6e18487 | 139 | @@ -518,14 +512,41 @@ static void recv_generator(char *fname, |
77945073 WD |
140 | if (verbose > 2) |
141 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
142 | ||
0bd6d59f | 143 | - write_int(f_out,i); |
dc3ae351 | 144 | - generate_and_send_sums(fd, st.st_size, f_out); |
d24c084d | 145 | +notify_others: |
d6e18487 | 146 | + if (f_out_name >= 0) { |
0b32df42 | 147 | + uchar lenbuf[3], *lb = lenbuf; |
0bd6d59f | 148 | + int len = statret == -1 ? 0 : strlen(fnamecmpbuf); |
0bd6d59f WD |
149 | + if (len > 0x7F) { |
150 | +#if MAXPATHLEN > 0x7FFF | |
151 | + *lb++ = len / 0x10000 + 0x80; | |
152 | + *lb++ = len / 0x100; | |
0b32df42 | 153 | +#else |
0bd6d59f | 154 | + *lb++ = len / 0x100 + 0x80; |
0b32df42 WD |
155 | +#endif |
156 | + } | |
157 | + *lb = len; | |
d6e18487 | 158 | + write_buf(f_out_name, lenbuf, lb - lenbuf + 1); |
0bd6d59f | 159 | + if (len) |
d6e18487 WD |
160 | + write_buf(f_out_name, fnamecmpbuf, len); |
161 | + io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */ | |
0b32df42 | 162 | + } |
0bd6d59f WD |
163 | + |
164 | + write_int(f_out, i); | |
2f5fa77e | 165 | + |
d6e18487 | 166 | + if (dry_run || read_batch) |
2f5fa77e WD |
167 | + return; |
168 | ||
169 | - close(fd); | |
0bd6d59f | 170 | + if (statret == 0) { |
dc3ae351 | 171 | + generate_and_send_sums(fd, st.st_size, f_out); |
0bd6d59f WD |
172 | + |
173 | + close(fd); | |
d6e18487 | 174 | + } else |
0bd6d59f | 175 | + write_sum_head(f_out, NULL); |
77945073 WD |
176 | } |
177 | ||
178 | ||
9be39c35 WD |
179 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
180 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
d6e18487 | 181 | + int f_out_name) |
77945073 WD |
182 | { |
183 | int i; | |
8cec1ead | 184 | int phase = 0; |
d6e18487 | 185 | @@ -566,7 +587,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
186 | } |
187 | ||
188 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 189 | - file, i, f_out); |
d6e18487 | 190 | + file, i, f_out, f_out_name); |
77945073 WD |
191 | } |
192 | ||
193 | phase++; | |
d6e18487 | 194 | @@ -583,7 +604,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
195 | while ((i = get_redo_num()) != -1) { |
196 | struct file_struct *file = flist->files[i]; | |
197 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 198 | - file, i, f_out); |
d6e18487 | 199 | + file, i, f_out, f_out_name); |
77945073 WD |
200 | } |
201 | ||
202 | phase++; | |
d6e18487 | 203 | @@ -602,7 +623,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
204 | if (!file->basename || !S_ISDIR(file->mode)) |
205 | continue; | |
206 | recv_generator(local_name ? local_name : f_name(file), | |
207 | - file, i, -1); | |
208 | + file, i, -1, -1); | |
209 | } | |
210 | ||
211 | if (verbose > 2) | |
d6e18487 | 212 | --- orig/main.c 2004-07-23 17:16:13 |
2f5fa77e WD |
213 | +++ main.c 2004-07-22 00:10:43 |
214 | @@ -58,6 +58,7 @@ extern int filesfrom_fd; | |
0bd6d59f WD |
215 | extern pid_t cleanup_child_pid; |
216 | extern char *files_from; | |
217 | extern char *remote_filesfrom_file; | |
218 | +extern char *compare_dest; | |
219 | extern char *rsync_path; | |
220 | extern char *shell_cmd; | |
221 | extern char *batch_name; | |
d6e18487 | 222 | @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
223 | { |
224 | int pid; | |
8cec1ead | 225 | int status = 0; |
77945073 | 226 | - int error_pipe[2]; |
77945073 | 227 | + int error_pipe[2], name_pipe[2]; |
d6e18487 | 228 | + BOOL need_name_pipe = compare_dest && !dry_run; |
77945073 WD |
229 | |
230 | if (preserve_hard_links) | |
231 | init_hard_links(flist); | |
d6e18487 | 232 | @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st |
d24c084d | 233 | delete_files(flist); |
77945073 WD |
234 | } |
235 | ||
236 | - if (fd_pair(error_pipe) < 0) { | |
237 | - rprintf(FERROR,"error pipe failed in do_recv\n"); | |
0bd6d59f WD |
238 | + if (fd_pair(error_pipe) < 0 |
239 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
77945073 WD |
240 | + rprintf(FERROR, "fd_pair() failed in do_recv\n"); |
241 | exit_cleanup(RERR_SOCKETIO); | |
242 | } | |
243 | ||
d6e18487 | 244 | @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 245 | |
8cec1ead | 246 | if ((pid = do_fork()) == 0) { |
77945073 | 247 | close(error_pipe[0]); |
0bd6d59f WD |
248 | + if (need_name_pipe) { |
249 | + close(name_pipe[1]); | |
250 | + set_blocking(name_pipe[0]); | |
251 | + } else | |
252 | + name_pipe[0] = -1; | |
77945073 WD |
253 | if (f_in != f_out) |
254 | close(f_out); | |
77945073 | 255 | |
d6e18487 | 256 | @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
257 | /* set place to send errors */ |
258 | set_msg_fd_out(error_pipe[1]); | |
259 | ||
260 | - recv_files(f_in,flist,local_name); | |
261 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
262 | io_flush(FULL_FLUSH); | |
263 | report(f_in); | |
264 | ||
d6e18487 | 265 | @@ -503,6 +511,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 266 | stop_write_batch(); |
77945073 WD |
267 | |
268 | close(error_pipe[1]); | |
0bd6d59f WD |
269 | + if (need_name_pipe) { |
270 | + close(name_pipe[0]); | |
271 | + set_nonblocking(name_pipe[1]); | |
272 | + } else | |
273 | + name_pipe[1] = -1; | |
77945073 WD |
274 | if (f_in != f_out) |
275 | close(f_in); | |
77945073 | 276 | |
d6e18487 | 277 | @@ -510,7 +523,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
278 | |
279 | set_msg_fd_in(error_pipe[0]); | |
280 | ||
281 | - generate_files(f_out, flist, local_name); | |
282 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
283 | ||
284 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
285 | report(-1); | |
d6e18487 | 286 | --- orig/receiver.c 2004-07-23 17:16:13 |
2f5fa77e WD |
287 | +++ receiver.c 2004-07-22 00:20:50 |
288 | @@ -320,6 +320,30 @@ static int receive_data(int f_in, char * | |
0bd6d59f WD |
289 | } |
290 | ||
5823d322 | 291 | |
0bd6d59f WD |
292 | +static char *read_gen_name(int fd, char *buf, char *realname) |
293 | +{ | |
294 | + int len = read_byte(fd); | |
295 | + if (len & 0x80) { | |
296 | +#if MAXPATHLEN > 32767 | |
297 | + uchar lenbuf[2]; | |
298 | + read_buf(fd, (char *)lenbuf, 2); | |
299 | + len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; | |
300 | +#else | |
301 | + len = (len & ~0x80) * 0x100 + read_byte(fd); | |
302 | +#endif | |
303 | + } | |
304 | + if (len) { | |
305 | + if (len >= MAXPATHLEN) { | |
306 | + rprintf(FERROR, "bogus data on generator name pipe\n"); | |
307 | + exit_cleanup(RERR_PROTOCOL); | |
308 | + } | |
309 | + read_sbuf(fd, buf, len); | |
310 | + return buf; | |
311 | + } | |
312 | + return realname; | |
313 | +} | |
314 | + | |
5823d322 WD |
315 | + |
316 | static void discard_receive_data(int f_in, OFF_T length) | |
317 | { | |
dc3ae351 | 318 | receive_data(f_in, NULL, -1, 0, NULL, -1, length); |
2f5fa77e | 319 | @@ -330,7 +354,8 @@ static void discard_receive_data(int f_i |
77945073 WD |
320 | * main routine for receiver process. |
321 | * | |
322 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 323 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 324 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
d6e18487 | 325 | + int f_in_name) |
77945073 | 326 | { |
2f5fa77e | 327 | int next_gen_i = -1; |
77945073 | 328 | int fd1,fd2; |
2f5fa77e | 329 | @@ -359,8 +384,15 @@ int recv_files(int f_in, struct file_lis |
0bd6d59f WD |
330 | i = read_int(f_in); |
331 | if (i == -1) { | |
2f5fa77e WD |
332 | if (read_batch) { |
333 | - if (next_gen_i != flist->count) | |
334 | - while (read_int(batch_gen_fd) != -1) {} | |
0bd6d59f | 335 | + while (next_gen_i < flist->count) { |
d6e18487 WD |
336 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
337 | + read_gen_name(f_in_name, | |
2f5fa77e WD |
338 | + fnamecmpbuf, NULL); |
339 | + } | |
d6e18487 | 340 | + next_gen_i = read_int(batch_gen_fd); |
2f5fa77e WD |
341 | + if (next_gen_i < 0) |
342 | + break; | |
0bd6d59f | 343 | + } |
2f5fa77e WD |
344 | next_gen_i = -1; |
345 | } | |
5823d322 | 346 | |
2f5fa77e | 347 | @@ -406,10 +438,12 @@ int recv_files(int f_in, struct file_lis |
77945073 WD |
348 | if (verbose > 2) |
349 | rprintf(FINFO,"recv_files(%s)\n",fname); | |
350 | ||
351 | - fnamecmp = fname; | |
2f5fa77e WD |
352 | - |
353 | if (read_batch) { | |
354 | while (i > next_gen_i) { | |
d6e18487 WD |
355 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
356 | + read_gen_name(f_in_name, fnamecmpbuf, | |
2f5fa77e WD |
357 | + NULL); |
358 | + } | |
359 | next_gen_i = read_int(batch_gen_fd); | |
360 | if (next_gen_i == -1) | |
361 | next_gen_i = flist->count; | |
362 | @@ -420,8 +454,14 @@ int recv_files(int f_in, struct file_lis | |
363 | discard_receive_data(f_in, file->length); | |
364 | continue; | |
365 | } | |
0bd6d59f | 366 | + next_gen_i = -1; |
2f5fa77e WD |
367 | } |
368 | ||
d6e18487 WD |
369 | + if (f_in_name >= 0) |
370 | + fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname); | |
0bd6d59f | 371 | + else |
0b32df42 | 372 | + fnamecmp = fname; |
2f5fa77e | 373 | + |
5823d322 WD |
374 | if (server_exclude_list.head |
375 | && check_exclude(&server_exclude_list, fname, | |
2f5fa77e WD |
376 | S_ISDIR(file->mode)) < 0) { |
377 | @@ -437,14 +477,6 @@ int recv_files(int f_in, struct file_lis | |
77945073 WD |
378 | /* open the file */ |
379 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
380 | ||
381 | - if (fd1 == -1 && compare_dest != NULL) { | |
382 | - /* try the file at compare_dest instead */ | |
383 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
384 | - compare_dest, fname); | |
385 | - fnamecmp = fnamecmpbuf; | |
386 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
387 | - } | |
388 | - | |
389 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
390 | rsyserr(FERROR, errno, "fstat %s failed", | |
391 | full_fname(fnamecmp)); |