Commit | Line | Data |
---|---|---|
77945073 WD |
1 | Added a pipe from the generator to the receiver that communicates |
2 | what basis file we used to generate the file data (if it was not | |
3 | the default name). This optimizes away the basis-file search in | |
4 | the receiver and makes future options that do more basis-file | |
5 | searching more efficient (such as the --fuzzy option and the | |
6 | support for multiple --compare-dest options). | |
7 | ||
0bd6d59f WD |
8 | Also fixes a potential synchronization problem between the generator |
9 | and the receiver in read-batch mode. Should consider making the | |
10 | sending of the index value the default for this pipe (it's currently | |
11 | only sent in batch mode due to the sender not listening to the | |
12 | generator to determine what files get updated). | |
13 | ||
77945073 WD |
14 | You must run "make proto" before compiling. |
15 | ||
dc3ae351 | 16 | --- orig/generator.c 2004-07-20 21:36:07 |
0bd6d59f | 17 | +++ generator.c 2004-07-17 15:50:09 |
dc3ae351 | 18 | @@ -258,9 +258,9 @@ static void generate_and_send_sums(int f |
77945073 WD |
19 | * out. It might be wrong. |
20 | */ | |
21 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
22 | - int f_out) | |
23 | + int f_out, int f_nameout) | |
24 | { | |
0bd6d59f WD |
25 | - int fd; |
26 | + int fd = -1; | |
77945073 | 27 | STRUCT_STAT st; |
0bd6d59f WD |
28 | int statret; |
29 | char *fnamecmp; | |
dc3ae351 | 30 | @@ -404,9 +404,6 @@ static void recv_generator(char *fname, |
0bd6d59f WD |
31 | } |
32 | #endif | |
33 | ||
34 | - if (read_batch) | |
35 | - return; | |
36 | - | |
37 | if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER)) | |
38 | return; | |
39 | ||
dc3ae351 | 40 | @@ -424,8 +421,10 @@ static void recv_generator(char *fname, |
77945073 WD |
41 | statret = link_stat(fnamecmpbuf, &st, 0); |
42 | if (!S_ISREG(st.st_mode)) | |
43 | statret = -1; | |
44 | - if (statret == -1) | |
0bd6d59f | 45 | + if (statret == -1) { |
77945073 WD |
46 | errno = saveerrno; |
47 | + *fnamecmpbuf = '\0'; | |
48 | + } | |
49 | #if HAVE_LINK | |
50 | else if (link_dest && !dry_run) { | |
51 | if (do_link(fnamecmpbuf, fname) != 0) { | |
dc3ae351 | 52 | @@ -433,22 +432,22 @@ static void recv_generator(char *fname, |
77945073 WD |
53 | rsyserr(FINFO, errno, "link %s => %s", |
54 | fnamecmpbuf, fname); | |
55 | } | |
56 | - } | |
57 | - fnamecmp = fnamecmpbuf; | |
58 | + fnamecmp = fnamecmpbuf; | |
59 | + } else | |
60 | + *fnamecmpbuf = '\0'; | |
61 | } | |
62 | #endif | |
63 | else | |
64 | fnamecmp = fnamecmpbuf; | |
65 | - } | |
66 | + } else | |
67 | + *fnamecmpbuf = '\0'; | |
68 | ||
69 | if (statret == -1) { | |
70 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
71 | return; | |
0bd6d59f WD |
72 | - if (errno == ENOENT) { |
73 | - write_int(f_out,i); | |
74 | - if (!dry_run) | |
75 | - write_sum_head(f_out, NULL); | |
76 | - } else if (verbose > 1) { | |
77 | + if (errno == ENOENT) | |
d24c084d | 78 | + goto notify_others; |
0bd6d59f WD |
79 | + if (verbose > 1) { |
80 | rsyserr(FERROR, errno, | |
81 | "recv_generator: failed to open %s", | |
82 | full_fname(fname)); | |
dc3ae351 | 83 | @@ -457,26 +456,23 @@ static void recv_generator(char *fname, |
0bd6d59f WD |
84 | } |
85 | ||
86 | if (!S_ISREG(st.st_mode)) { | |
87 | - if (delete_file(fname) != 0) { | |
88 | + if (delete_file(fname) != 0) | |
89 | return; | |
90 | - } | |
91 | ||
77945073 WD |
92 | /* now pretend the file didn't exist */ |
93 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
94 | return; | |
0bd6d59f WD |
95 | - write_int(f_out,i); |
96 | - if (!dry_run) | |
97 | - write_sum_head(f_out, NULL); | |
98 | - return; | |
99 | + statret = -1; | |
d24c084d | 100 | + goto notify_others; |
77945073 WD |
101 | } |
102 | ||
103 | - if (opt_ignore_existing && fnamecmp == fname) { | |
104 | + if (opt_ignore_existing && !*fnamecmpbuf) { | |
105 | if (verbose > 1) | |
106 | rprintf(FINFO,"%s exists\n",fname); | |
107 | return; | |
108 | } | |
109 | ||
110 | - if (update_only && fnamecmp == fname | |
111 | + if (update_only && !*fnamecmpbuf | |
112 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { | |
113 | if (verbose > 1) | |
114 | rprintf(FINFO,"%s is newer\n",fname); | |
dc3ae351 | 115 | @@ -484,21 +480,17 @@ static void recv_generator(char *fname, |
77945073 WD |
116 | } |
117 | ||
118 | if (skip_file(fname, file, &st)) { | |
119 | - if (fnamecmp == fname) | |
120 | + if (!*fnamecmpbuf) | |
121 | set_perms(fname, file, &st, PERMS_REPORT); | |
122 | return; | |
123 | } | |
124 | ||
0bd6d59f WD |
125 | - if (dry_run) { |
126 | - write_int(f_out,i); | |
127 | - return; | |
128 | - } | |
129 | - | |
130 | - if (whole_file > 0) { | |
131 | - write_int(f_out,i); | |
132 | - write_sum_head(f_out, NULL); | |
133 | - return; | |
134 | + if (dry_run || whole_file) { | |
135 | + statret = -1; | |
d24c084d | 136 | + goto notify_others; |
77945073 | 137 | } |
0bd6d59f | 138 | + if (read_batch) |
d24c084d | 139 | + goto notify_others; |
77945073 | 140 | |
0bd6d59f WD |
141 | /* open the file */ |
142 | fd = do_open(fnamecmp, O_RDONLY, 0); | |
dc3ae351 | 143 | @@ -509,9 +501,8 @@ static void recv_generator(char *fname, |
77945073 WD |
144 | /* pretend the file didn't exist */ |
145 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
146 | return; | |
0bd6d59f WD |
147 | - write_int(f_out,i); |
148 | - write_sum_head(f_out, NULL); | |
149 | - return; | |
150 | + statret = -1; | |
d24c084d | 151 | + goto notify_others; |
0bd6d59f WD |
152 | } |
153 | ||
0bd6d59f | 154 | if (verbose > 3) { |
dc3ae351 | 155 | @@ -522,14 +513,41 @@ static void recv_generator(char *fname, |
77945073 WD |
156 | if (verbose > 2) |
157 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
158 | ||
0bd6d59f | 159 | - write_int(f_out,i); |
dc3ae351 | 160 | - generate_and_send_sums(fd, st.st_size, f_out); |
d24c084d | 161 | +notify_others: |
0b32df42 WD |
162 | + if (f_nameout >= 0) { |
163 | + uchar lenbuf[3], *lb = lenbuf; | |
0bd6d59f WD |
164 | + int len = statret == -1 ? 0 : strlen(fnamecmpbuf); |
165 | + if (read_batch) | |
166 | + write_int(f_nameout, i); | |
167 | + if (len > 0x7F) { | |
168 | +#if MAXPATHLEN > 0x7FFF | |
169 | + *lb++ = len / 0x10000 + 0x80; | |
170 | + *lb++ = len / 0x100; | |
0b32df42 | 171 | +#else |
0bd6d59f | 172 | + *lb++ = len / 0x100 + 0x80; |
0b32df42 WD |
173 | +#endif |
174 | + } | |
175 | + *lb = len; | |
176 | + write_buf(f_nameout, lenbuf, lb - lenbuf + 1); | |
0bd6d59f WD |
177 | + if (len) |
178 | + write_buf(f_nameout, fnamecmpbuf, len); | |
0b32df42 | 179 | + } |
77945073 | 180 | |
0bd6d59f | 181 | - close(fd); |
0bd6d59f WD |
182 | + if (read_batch) |
183 | + return; | |
184 | + | |
185 | + write_int(f_out, i); | |
186 | + if (statret == 0) { | |
dc3ae351 | 187 | + generate_and_send_sums(fd, st.st_size, f_out); |
0bd6d59f WD |
188 | + |
189 | + close(fd); | |
0bd6d59f WD |
190 | + } else if (!dry_run) |
191 | + write_sum_head(f_out, NULL); | |
77945073 WD |
192 | } |
193 | ||
194 | ||
9be39c35 WD |
195 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
196 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
77945073 WD |
197 | + int f_nameout) |
198 | { | |
199 | int i; | |
8cec1ead | 200 | int phase = 0; |
dc3ae351 | 201 | @@ -570,7 +588,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
202 | } |
203 | ||
204 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 WD |
205 | - file, i, f_out); |
206 | + file, i, f_out, f_nameout); | |
77945073 WD |
207 | } |
208 | ||
209 | phase++; | |
dc3ae351 | 210 | @@ -581,13 +599,15 @@ void generate_files(int f_out, struct fi |
0bd6d59f WD |
211 | rprintf(FINFO,"generate_files phase=%d\n",phase); |
212 | ||
213 | write_int(f_out, -1); | |
214 | + if (read_batch) | |
215 | + write_int(f_nameout, flist->count); | |
216 | ||
217 | /* files can cycle through the system more than once | |
218 | * to catch initial checksum errors */ | |
77945073 WD |
219 | while ((i = get_redo_num()) != -1) { |
220 | struct file_struct *file = flist->files[i]; | |
221 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 WD |
222 | - file, i, f_out); |
223 | + file, i, f_out, f_nameout); | |
77945073 WD |
224 | } |
225 | ||
226 | phase++; | |
dc3ae351 | 227 | @@ -595,6 +615,8 @@ void generate_files(int f_out, struct fi |
0bd6d59f WD |
228 | rprintf(FINFO,"generate_files phase=%d\n",phase); |
229 | ||
230 | write_int(f_out, -1); | |
231 | + if (read_batch) | |
232 | + write_int(f_nameout, flist->count); | |
233 | ||
234 | if (preserve_hard_links) | |
235 | do_hard_links(); | |
dc3ae351 | 236 | @@ -606,7 +628,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
237 | if (!file->basename || !S_ISDIR(file->mode)) |
238 | continue; | |
239 | recv_generator(local_name ? local_name : f_name(file), | |
240 | - file, i, -1); | |
241 | + file, i, -1, -1); | |
242 | } | |
243 | ||
244 | if (verbose > 2) | |
5823d322 | 245 | --- orig/main.c 2004-07-19 17:14:44 |
0bd6d59f WD |
246 | +++ main.c 2004-07-17 15:58:11 |
247 | @@ -57,6 +57,7 @@ extern int filesfrom_fd; | |
248 | extern pid_t cleanup_child_pid; | |
249 | extern char *files_from; | |
250 | extern char *remote_filesfrom_file; | |
251 | +extern char *compare_dest; | |
252 | extern char *rsync_path; | |
253 | extern char *shell_cmd; | |
254 | extern char *batch_name; | |
d24c084d | 255 | @@ -444,20 +445,21 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
256 | { |
257 | int pid; | |
8cec1ead | 258 | int status = 0; |
77945073 | 259 | - int error_pipe[2]; |
77945073 | 260 | + int error_pipe[2], name_pipe[2]; |
0bd6d59f | 261 | + int need_name_pipe = compare_dest || read_batch; |
77945073 WD |
262 | |
263 | if (preserve_hard_links) | |
264 | init_hard_links(flist); | |
d24c084d WD |
265 | |
266 | if (!delete_after) { | |
267 | /* I moved this here from recv_files() to prevent a race condition */ | |
268 | - if (recurse && delete_mode && !local_name && flist->count>0) { | |
269 | + if (recurse && delete_mode && !local_name && flist->count > 0) | |
270 | delete_files(flist); | |
271 | - } | |
77945073 WD |
272 | } |
273 | ||
274 | - if (fd_pair(error_pipe) < 0) { | |
275 | - rprintf(FERROR,"error pipe failed in do_recv\n"); | |
0bd6d59f WD |
276 | + if (fd_pair(error_pipe) < 0 |
277 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
77945073 WD |
278 | + rprintf(FERROR, "fd_pair() failed in do_recv\n"); |
279 | exit_cleanup(RERR_SOCKETIO); | |
280 | } | |
281 | ||
d24c084d | 282 | @@ -465,6 +467,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 283 | |
8cec1ead | 284 | if ((pid = do_fork()) == 0) { |
77945073 | 285 | close(error_pipe[0]); |
0bd6d59f WD |
286 | + if (need_name_pipe) { |
287 | + close(name_pipe[1]); | |
288 | + set_blocking(name_pipe[0]); | |
289 | + } else | |
290 | + name_pipe[0] = -1; | |
77945073 WD |
291 | if (f_in != f_out) |
292 | close(f_out); | |
77945073 | 293 | |
d24c084d | 294 | @@ -474,7 +481,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
295 | /* set place to send errors */ |
296 | set_msg_fd_out(error_pipe[1]); | |
297 | ||
298 | - recv_files(f_in,flist,local_name); | |
299 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
300 | io_flush(FULL_FLUSH); | |
301 | report(f_in); | |
302 | ||
d24c084d | 303 | @@ -492,6 +499,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 304 | stop_write_batch(); |
77945073 WD |
305 | |
306 | close(error_pipe[1]); | |
0bd6d59f WD |
307 | + if (need_name_pipe) { |
308 | + close(name_pipe[0]); | |
309 | + set_nonblocking(name_pipe[1]); | |
310 | + } else | |
311 | + name_pipe[1] = -1; | |
77945073 WD |
312 | if (f_in != f_out) |
313 | close(f_in); | |
77945073 | 314 | |
d24c084d | 315 | @@ -499,7 +511,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
316 | |
317 | set_msg_fd_in(error_pipe[0]); | |
318 | ||
319 | - generate_files(f_out, flist, local_name); | |
320 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
321 | ||
322 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
323 | report(-1); | |
dc3ae351 | 324 | --- orig/receiver.c 2004-07-20 21:36:07 |
5823d322 | 325 | +++ receiver.c 2004-07-19 16:44:39 |
0bd6d59f WD |
326 | @@ -28,6 +28,7 @@ extern int max_delete; |
327 | extern int csum_length; | |
328 | extern struct stats stats; | |
329 | extern int dry_run; | |
330 | +extern int read_batch; | |
331 | extern int am_server; | |
332 | extern int relative_paths; | |
333 | extern int keep_dirlinks; | |
dc3ae351 | 334 | @@ -318,6 +319,30 @@ static int receive_data(int f_in, char * |
0bd6d59f WD |
335 | } |
336 | ||
5823d322 | 337 | |
0bd6d59f WD |
338 | +static char *read_gen_name(int fd, char *buf, char *realname) |
339 | +{ | |
340 | + int len = read_byte(fd); | |
341 | + if (len & 0x80) { | |
342 | +#if MAXPATHLEN > 32767 | |
343 | + uchar lenbuf[2]; | |
344 | + read_buf(fd, (char *)lenbuf, 2); | |
345 | + len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; | |
346 | +#else | |
347 | + len = (len & ~0x80) * 0x100 + read_byte(fd); | |
348 | +#endif | |
349 | + } | |
350 | + if (len) { | |
351 | + if (len >= MAXPATHLEN) { | |
352 | + rprintf(FERROR, "bogus data on generator name pipe\n"); | |
353 | + exit_cleanup(RERR_PROTOCOL); | |
354 | + } | |
355 | + read_sbuf(fd, buf, len); | |
356 | + return buf; | |
357 | + } | |
358 | + return realname; | |
359 | +} | |
360 | + | |
5823d322 WD |
361 | + |
362 | static void discard_receive_data(int f_in, OFF_T length) | |
363 | { | |
dc3ae351 WD |
364 | receive_data(f_in, NULL, -1, 0, NULL, -1, length); |
365 | @@ -328,8 +353,10 @@ static void discard_receive_data(int f_i | |
77945073 WD |
366 | * main routine for receiver process. |
367 | * | |
368 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 369 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 370 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
0bd6d59f | 371 | + int f_name_in) |
77945073 | 372 | { |
0bd6d59f | 373 | + int next_gen_i = -1; |
77945073 WD |
374 | int fd1,fd2; |
375 | STRUCT_STAT st; | |
0bd6d59f | 376 | char *fname, fbuf[MAXPATHLEN]; |
dc3ae351 | 377 | @@ -355,6 +382,17 @@ int recv_files(int f_in, struct file_lis |
0b32df42 | 378 | |
0bd6d59f WD |
379 | i = read_int(f_in); |
380 | if (i == -1) { | |
381 | + if (read_batch) { | |
382 | + if (next_gen_i < 0) | |
383 | + next_gen_i = read_int(f_name_in); | |
384 | + while (next_gen_i < flist->count) { | |
385 | + read_gen_name(f_name_in, fnamecmpbuf, | |
386 | + NULL); | |
387 | + next_gen_i = read_int(f_name_in); | |
388 | + } | |
389 | + next_gen_i = -1; | |
390 | + } | |
391 | + | |
392 | if (phase) | |
393 | break; | |
5823d322 | 394 | |
dc3ae351 | 395 | @@ -397,7 +435,26 @@ int recv_files(int f_in, struct file_lis |
77945073 WD |
396 | if (verbose > 2) |
397 | rprintf(FINFO,"recv_files(%s)\n",fname); | |
398 | ||
399 | - fnamecmp = fname; | |
0bd6d59f WD |
400 | + if (read_batch) { |
401 | + if (next_gen_i < 0) | |
402 | + next_gen_i = read_int(f_name_in); | |
403 | + while (i > next_gen_i) { | |
404 | + read_gen_name(f_name_in, fnamecmpbuf, NULL); | |
405 | + next_gen_i = read_int(f_name_in); | |
406 | + } | |
407 | + if (i < next_gen_i) { | |
5823d322 | 408 | + rprintf(FINFO, "skipping update for \"%s\"\n", |
0bd6d59f | 409 | + fname); |
5823d322 | 410 | + discard_receive_data(f_in, file->length); |
0bd6d59f WD |
411 | + continue; |
412 | + } | |
413 | + next_gen_i = -1; | |
77945073 | 414 | + } |
0bd6d59f WD |
415 | + |
416 | + if (f_name_in >= 0) | |
417 | + fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname); | |
418 | + else | |
0b32df42 | 419 | + fnamecmp = fname; |
77945073 | 420 | |
5823d322 WD |
421 | if (server_exclude_list.head |
422 | && check_exclude(&server_exclude_list, fname, | |
dc3ae351 | 423 | @@ -414,14 +471,6 @@ int recv_files(int f_in, struct file_lis |
77945073 WD |
424 | /* open the file */ |
425 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
426 | ||
427 | - if (fd1 == -1 && compare_dest != NULL) { | |
428 | - /* try the file at compare_dest instead */ | |
429 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
430 | - compare_dest, fname); | |
431 | - fnamecmp = fnamecmpbuf; | |
432 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
433 | - } | |
434 | - | |
435 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
436 | rsyserr(FERROR, errno, "fstat %s failed", | |
437 | full_fname(fnamecmp)); |