Commit | Line | Data |
---|---|---|
77945073 WD |
1 | Added a pipe from the generator to the receiver that communicates |
2 | what basis file we used to generate the file data (if it was not | |
3 | the default name). This optimizes away the basis-file search in | |
4 | the receiver and makes future options that do more basis-file | |
5 | searching more efficient (such as the --fuzzy option and the | |
6 | support for multiple --compare-dest options). | |
7 | ||
0bd6d59f WD |
8 | Also fixes a potential synchronization problem between the generator |
9 | and the receiver in read-batch mode. Should consider making the | |
10 | sending of the index value the default for this pipe (it's currently | |
11 | only sent in batch mode due to the sender not listening to the | |
12 | generator to determine what files get updated). | |
13 | ||
77945073 WD |
14 | You must run "make proto" before compiling. |
15 | ||
0bd6d59f WD |
16 | --- orig/generator.c 2004-07-17 16:30:20 |
17 | +++ generator.c 2004-07-17 15:50:09 | |
18 | @@ -251,11 +251,11 @@ static void generate_and_send_sums(struc | |
77945073 WD |
19 | * out. It might be wrong. |
20 | */ | |
21 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
22 | - int f_out) | |
23 | + int f_out, int f_nameout) | |
24 | { | |
0bd6d59f WD |
25 | - int fd; |
26 | + int fd = -1; | |
77945073 | 27 | STRUCT_STAT st; |
0bd6d59f WD |
28 | - struct map_struct *mapbuf; |
29 | + struct map_struct *mapbuf = NULL; | |
30 | int statret; | |
31 | char *fnamecmp; | |
32 | char fnamecmpbuf[MAXPATHLEN]; | |
33 | @@ -398,9 +398,6 @@ static void recv_generator(char *fname, | |
34 | } | |
35 | #endif | |
36 | ||
37 | - if (read_batch) | |
38 | - return; | |
39 | - | |
40 | if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER)) | |
41 | return; | |
42 | ||
43 | @@ -418,8 +415,10 @@ static void recv_generator(char *fname, | |
77945073 WD |
44 | statret = link_stat(fnamecmpbuf, &st, 0); |
45 | if (!S_ISREG(st.st_mode)) | |
46 | statret = -1; | |
47 | - if (statret == -1) | |
0bd6d59f | 48 | + if (statret == -1) { |
77945073 WD |
49 | errno = saveerrno; |
50 | + *fnamecmpbuf = '\0'; | |
51 | + } | |
52 | #if HAVE_LINK | |
53 | else if (link_dest && !dry_run) { | |
54 | if (do_link(fnamecmpbuf, fname) != 0) { | |
0bd6d59f | 55 | @@ -427,22 +426,22 @@ static void recv_generator(char *fname, |
77945073 WD |
56 | rsyserr(FINFO, errno, "link %s => %s", |
57 | fnamecmpbuf, fname); | |
58 | } | |
59 | - } | |
60 | - fnamecmp = fnamecmpbuf; | |
61 | + fnamecmp = fnamecmpbuf; | |
62 | + } else | |
63 | + *fnamecmpbuf = '\0'; | |
64 | } | |
65 | #endif | |
66 | else | |
67 | fnamecmp = fnamecmpbuf; | |
68 | - } | |
69 | + } else | |
70 | + *fnamecmpbuf = '\0'; | |
71 | ||
72 | if (statret == -1) { | |
73 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
74 | return; | |
0bd6d59f WD |
75 | - if (errno == ENOENT) { |
76 | - write_int(f_out,i); | |
77 | - if (!dry_run) | |
78 | - write_sum_head(f_out, NULL); | |
79 | - } else if (verbose > 1) { | |
80 | + if (errno == ENOENT) | |
81 | + goto notify_sender; | |
82 | + if (verbose > 1) { | |
83 | rsyserr(FERROR, errno, | |
84 | "recv_generator: failed to open %s", | |
85 | full_fname(fname)); | |
86 | @@ -451,26 +450,23 @@ static void recv_generator(char *fname, | |
87 | } | |
88 | ||
89 | if (!S_ISREG(st.st_mode)) { | |
90 | - if (delete_file(fname) != 0) { | |
91 | + if (delete_file(fname) != 0) | |
92 | return; | |
93 | - } | |
94 | ||
77945073 WD |
95 | /* now pretend the file didn't exist */ |
96 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
97 | return; | |
0bd6d59f WD |
98 | - write_int(f_out,i); |
99 | - if (!dry_run) | |
100 | - write_sum_head(f_out, NULL); | |
101 | - return; | |
102 | + statret = -1; | |
103 | + goto notify_sender; | |
77945073 WD |
104 | } |
105 | ||
106 | - if (opt_ignore_existing && fnamecmp == fname) { | |
107 | + if (opt_ignore_existing && !*fnamecmpbuf) { | |
108 | if (verbose > 1) | |
109 | rprintf(FINFO,"%s exists\n",fname); | |
110 | return; | |
111 | } | |
112 | ||
113 | - if (update_only && fnamecmp == fname | |
114 | + if (update_only && !*fnamecmpbuf | |
115 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { | |
116 | if (verbose > 1) | |
117 | rprintf(FINFO,"%s is newer\n",fname); | |
0bd6d59f | 118 | @@ -478,21 +474,17 @@ static void recv_generator(char *fname, |
77945073 WD |
119 | } |
120 | ||
121 | if (skip_file(fname, file, &st)) { | |
122 | - if (fnamecmp == fname) | |
123 | + if (!*fnamecmpbuf) | |
124 | set_perms(fname, file, &st, PERMS_REPORT); | |
125 | return; | |
126 | } | |
127 | ||
0bd6d59f WD |
128 | - if (dry_run) { |
129 | - write_int(f_out,i); | |
130 | - return; | |
131 | - } | |
132 | - | |
133 | - if (whole_file > 0) { | |
134 | - write_int(f_out,i); | |
135 | - write_sum_head(f_out, NULL); | |
136 | - return; | |
137 | + if (dry_run || whole_file) { | |
138 | + statret = -1; | |
139 | + goto notify_sender; | |
77945073 | 140 | } |
0bd6d59f WD |
141 | + if (read_batch) |
142 | + goto notify_sender; | |
77945073 | 143 | |
0bd6d59f WD |
144 | /* open the file */ |
145 | fd = do_open(fnamecmp, O_RDONLY, 0); | |
146 | @@ -503,15 +495,12 @@ static void recv_generator(char *fname, | |
77945073 WD |
147 | /* pretend the file didn't exist */ |
148 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
149 | return; | |
0bd6d59f WD |
150 | - write_int(f_out,i); |
151 | - write_sum_head(f_out, NULL); | |
152 | - return; | |
153 | + statret = -1; | |
154 | + goto notify_sender; | |
155 | } | |
156 | ||
157 | if (st.st_size > 0) | |
158 | mapbuf = map_file(fd,st.st_size); | |
159 | - else | |
160 | - mapbuf = NULL; | |
161 | ||
162 | if (verbose > 3) { | |
163 | rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp, | |
164 | @@ -521,16 +510,43 @@ static void recv_generator(char *fname, | |
77945073 WD |
165 | if (verbose > 2) |
166 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
167 | ||
0bd6d59f WD |
168 | - write_int(f_out,i); |
169 | - generate_and_send_sums(mapbuf, st.st_size, f_out); | |
170 | +notify_sender: | |
0b32df42 WD |
171 | + if (f_nameout >= 0) { |
172 | + uchar lenbuf[3], *lb = lenbuf; | |
0bd6d59f WD |
173 | + int len = statret == -1 ? 0 : strlen(fnamecmpbuf); |
174 | + if (read_batch) | |
175 | + write_int(f_nameout, i); | |
176 | + if (len > 0x7F) { | |
177 | +#if MAXPATHLEN > 0x7FFF | |
178 | + *lb++ = len / 0x10000 + 0x80; | |
179 | + *lb++ = len / 0x100; | |
0b32df42 | 180 | +#else |
0bd6d59f | 181 | + *lb++ = len / 0x100 + 0x80; |
0b32df42 WD |
182 | +#endif |
183 | + } | |
184 | + *lb = len; | |
185 | + write_buf(f_nameout, lenbuf, lb - lenbuf + 1); | |
0bd6d59f WD |
186 | + if (len) |
187 | + write_buf(f_nameout, fnamecmpbuf, len); | |
0b32df42 | 188 | + } |
77945073 | 189 | |
0bd6d59f WD |
190 | - close(fd); |
191 | - if (mapbuf) | |
192 | - unmap_file(mapbuf); | |
193 | + if (read_batch) | |
194 | + return; | |
195 | + | |
196 | + write_int(f_out, i); | |
197 | + if (statret == 0) { | |
198 | + generate_and_send_sums(mapbuf, st.st_size, f_out); | |
199 | + | |
200 | + close(fd); | |
201 | + if (mapbuf) | |
202 | + unmap_file(mapbuf); | |
203 | + } else if (!dry_run) | |
204 | + write_sum_head(f_out, NULL); | |
77945073 WD |
205 | } |
206 | ||
207 | ||
9be39c35 WD |
208 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
209 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
77945073 WD |
210 | + int f_nameout) |
211 | { | |
212 | int i; | |
8cec1ead | 213 | int phase = 0; |
0bd6d59f | 214 | @@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
215 | } |
216 | ||
217 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 WD |
218 | - file, i, f_out); |
219 | + file, i, f_out, f_nameout); | |
77945073 WD |
220 | } |
221 | ||
222 | phase++; | |
0bd6d59f WD |
223 | @@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi |
224 | rprintf(FINFO,"generate_files phase=%d\n",phase); | |
225 | ||
226 | write_int(f_out, -1); | |
227 | + if (read_batch) | |
228 | + write_int(f_nameout, flist->count); | |
229 | ||
230 | /* files can cycle through the system more than once | |
231 | * to catch initial checksum errors */ | |
77945073 WD |
232 | while ((i = get_redo_num()) != -1) { |
233 | struct file_struct *file = flist->files[i]; | |
234 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 WD |
235 | - file, i, f_out); |
236 | + file, i, f_out, f_nameout); | |
77945073 WD |
237 | } |
238 | ||
239 | phase++; | |
0bd6d59f WD |
240 | @@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi |
241 | rprintf(FINFO,"generate_files phase=%d\n",phase); | |
242 | ||
243 | write_int(f_out, -1); | |
244 | + if (read_batch) | |
245 | + write_int(f_nameout, flist->count); | |
246 | ||
247 | if (preserve_hard_links) | |
248 | do_hard_links(); | |
249 | @@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi | |
77945073 WD |
250 | if (!file->basename || !S_ISDIR(file->mode)) |
251 | continue; | |
252 | recv_generator(local_name ? local_name : f_name(file), | |
253 | - file, i, -1); | |
254 | + file, i, -1, -1); | |
255 | } | |
256 | ||
257 | if (verbose > 2) | |
0b32df42 | 258 | --- orig/main.c 2004-07-17 15:20:05 |
0bd6d59f WD |
259 | +++ main.c 2004-07-17 15:58:11 |
260 | @@ -57,6 +57,7 @@ extern int filesfrom_fd; | |
261 | extern pid_t cleanup_child_pid; | |
262 | extern char *files_from; | |
263 | extern char *remote_filesfrom_file; | |
264 | +extern char *compare_dest; | |
265 | extern char *rsync_path; | |
266 | extern char *shell_cmd; | |
267 | extern char *batch_name; | |
268 | @@ -444,7 +445,8 @@ static int do_recv(int f_in,int f_out,st | |
77945073 WD |
269 | { |
270 | int pid; | |
8cec1ead | 271 | int status = 0; |
77945073 | 272 | - int error_pipe[2]; |
77945073 | 273 | + int error_pipe[2], name_pipe[2]; |
0bd6d59f | 274 | + int need_name_pipe = compare_dest || read_batch; |
77945073 WD |
275 | |
276 | if (preserve_hard_links) | |
277 | init_hard_links(flist); | |
0bd6d59f | 278 | @@ -456,8 +458,9 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
279 | } |
280 | } | |
281 | ||
282 | - if (fd_pair(error_pipe) < 0) { | |
283 | - rprintf(FERROR,"error pipe failed in do_recv\n"); | |
0bd6d59f WD |
284 | + if (fd_pair(error_pipe) < 0 |
285 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
77945073 WD |
286 | + rprintf(FERROR, "fd_pair() failed in do_recv\n"); |
287 | exit_cleanup(RERR_SOCKETIO); | |
288 | } | |
289 | ||
0bd6d59f | 290 | @@ -465,6 +468,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 291 | |
8cec1ead | 292 | if ((pid = do_fork()) == 0) { |
77945073 | 293 | close(error_pipe[0]); |
0bd6d59f WD |
294 | + if (need_name_pipe) { |
295 | + close(name_pipe[1]); | |
296 | + set_blocking(name_pipe[0]); | |
297 | + } else | |
298 | + name_pipe[0] = -1; | |
77945073 WD |
299 | if (f_in != f_out) |
300 | close(f_out); | |
77945073 | 301 | |
0bd6d59f | 302 | @@ -474,7 +482,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
303 | /* set place to send errors */ |
304 | set_msg_fd_out(error_pipe[1]); | |
305 | ||
306 | - recv_files(f_in,flist,local_name); | |
307 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
308 | io_flush(FULL_FLUSH); | |
309 | report(f_in); | |
310 | ||
0bd6d59f | 311 | @@ -492,6 +500,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 312 | stop_write_batch(); |
77945073 WD |
313 | |
314 | close(error_pipe[1]); | |
0bd6d59f WD |
315 | + if (need_name_pipe) { |
316 | + close(name_pipe[0]); | |
317 | + set_nonblocking(name_pipe[1]); | |
318 | + } else | |
319 | + name_pipe[1] = -1; | |
77945073 WD |
320 | if (f_in != f_out) |
321 | close(f_in); | |
77945073 | 322 | |
0bd6d59f | 323 | @@ -499,7 +512,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
324 | |
325 | set_msg_fd_in(error_pipe[0]); | |
326 | ||
327 | - generate_files(f_out, flist, local_name); | |
328 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
329 | ||
330 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
331 | report(-1); | |
f6c3b300 | 332 | --- orig/receiver.c 2004-07-16 20:07:22 |
0bd6d59f WD |
333 | +++ receiver.c 2004-07-17 21:27:55 |
334 | @@ -28,6 +28,7 @@ extern int max_delete; | |
335 | extern int csum_length; | |
336 | extern struct stats stats; | |
337 | extern int dry_run; | |
338 | +extern int read_batch; | |
339 | extern int am_server; | |
340 | extern int relative_paths; | |
341 | extern int keep_dirlinks; | |
342 | @@ -299,13 +300,38 @@ static int receive_data(int f_in,struct | |
343 | return 1; | |
344 | } | |
345 | ||
346 | +static char *read_gen_name(int fd, char *buf, char *realname) | |
347 | +{ | |
348 | + int len = read_byte(fd); | |
349 | + if (len & 0x80) { | |
350 | +#if MAXPATHLEN > 32767 | |
351 | + uchar lenbuf[2]; | |
352 | + read_buf(fd, (char *)lenbuf, 2); | |
353 | + len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; | |
354 | +#else | |
355 | + len = (len & ~0x80) * 0x100 + read_byte(fd); | |
356 | +#endif | |
357 | + } | |
358 | + if (len) { | |
359 | + if (len >= MAXPATHLEN) { | |
360 | + rprintf(FERROR, "bogus data on generator name pipe\n"); | |
361 | + exit_cleanup(RERR_PROTOCOL); | |
362 | + } | |
363 | + read_sbuf(fd, buf, len); | |
364 | + return buf; | |
365 | + } | |
366 | + return realname; | |
367 | +} | |
368 | + | |
369 | ||
370 | /** | |
77945073 WD |
371 | * main routine for receiver process. |
372 | * | |
373 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 374 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 375 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
0bd6d59f | 376 | + int f_name_in) |
77945073 | 377 | { |
0bd6d59f | 378 | + int next_gen_i = -1; |
77945073 WD |
379 | int fd1,fd2; |
380 | STRUCT_STAT st; | |
0bd6d59f WD |
381 | char *fname, fbuf[MAXPATHLEN]; |
382 | @@ -332,8 +358,20 @@ int recv_files(int f_in, struct file_lis | |
0b32df42 | 383 | |
0bd6d59f WD |
384 | i = read_int(f_in); |
385 | if (i == -1) { | |
386 | + if (read_batch) { | |
387 | + if (next_gen_i < 0) | |
388 | + next_gen_i = read_int(f_name_in); | |
389 | + while (next_gen_i < flist->count) { | |
390 | + read_gen_name(f_name_in, fnamecmpbuf, | |
391 | + NULL); | |
392 | + next_gen_i = read_int(f_name_in); | |
393 | + } | |
394 | + next_gen_i = -1; | |
395 | + } | |
396 | + | |
397 | if (phase) | |
398 | break; | |
399 | + | |
400 | phase = 1; | |
401 | csum_length = SUM_LENGTH; | |
402 | if (verbose > 2) | |
403 | @@ -373,19 +411,31 @@ int recv_files(int f_in, struct file_lis | |
77945073 WD |
404 | if (verbose > 2) |
405 | rprintf(FINFO,"recv_files(%s)\n",fname); | |
406 | ||
407 | - fnamecmp = fname; | |
0bd6d59f WD |
408 | + if (read_batch) { |
409 | + if (next_gen_i < 0) | |
410 | + next_gen_i = read_int(f_name_in); | |
411 | + while (i > next_gen_i) { | |
412 | + read_gen_name(f_name_in, fnamecmpbuf, NULL); | |
413 | + next_gen_i = read_int(f_name_in); | |
414 | + } | |
415 | + if (i < next_gen_i) { | |
416 | + rprintf(FINFO, "skipping update for %s\n", | |
417 | + fname); | |
418 | + receive_data(f_in,NULL,-1,NULL,file->length); | |
419 | + continue; | |
420 | + } | |
421 | + next_gen_i = -1; | |
77945073 | 422 | + } |
0bd6d59f WD |
423 | + |
424 | + if (f_name_in >= 0) | |
425 | + fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname); | |
426 | + else | |
0b32df42 | 427 | + fnamecmp = fname; |
0bd6d59f | 428 | + |
77945073 WD |
429 | |
430 | /* open the file */ | |
431 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
432 | ||
433 | - if (fd1 == -1 && compare_dest != NULL) { | |
434 | - /* try the file at compare_dest instead */ | |
435 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
436 | - compare_dest, fname); | |
437 | - fnamecmp = fnamecmpbuf; | |
438 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
439 | - } | |
440 | - | |
441 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
442 | rsyserr(FERROR, errno, "fstat %s failed", | |
443 | full_fname(fnamecmp)); |