Commit | Line | Data |
---|---|---|
77945073 WD |
1 | Added a pipe from the generator to the receiver that communicates |
2 | what basis file we used to generate the file data (if it was not | |
3 | the default name). This optimizes away the basis-file search in | |
4 | the receiver and makes future options that do more basis-file | |
5 | searching more efficient (such as the --fuzzy option and the | |
6 | support for multiple --compare-dest options). | |
7 | ||
8 | You must run "make proto" before compiling. | |
9 | ||
2eb7ca6c | 10 | --- orig/generator.c 2004-07-29 16:08:03 |
b952a177 WD |
11 | +++ generator.c 2004-07-28 10:14:15 |
12 | @@ -259,9 +259,9 @@ static void generate_and_send_sums(int f | |
77945073 WD |
13 | * out. It might be wrong. |
14 | */ | |
15 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
16 | - int f_out) | |
d6e18487 | 17 | + int f_out, int f_out_name) |
77945073 | 18 | { |
0bd6d59f WD |
19 | - int fd; |
20 | + int fd = -1; | |
77945073 | 21 | STRUCT_STAT st; |
b952a177 | 22 | int statret, stat_errno; |
0bd6d59f | 23 | char *fnamecmp; |
b952a177 WD |
24 | @@ -437,13 +437,16 @@ static void recv_generator(char *fname, |
25 | safe_fname(fname)); | |
26 | } | |
27 | fnamecmp = fnamecmpbuf; | |
28 | - } | |
29 | + } else | |
30 | + *fnamecmpbuf = '\0'; | |
31 | } else | |
77945073 | 32 | #endif |
b952a177 WD |
33 | fnamecmp = fnamecmpbuf; |
34 | statret = 0; | |
35 | - } | |
77945073 | 36 | - } |
b952a177 WD |
37 | + } else |
38 | + *fnamecmpbuf = '\0'; | |
77945073 WD |
39 | + } else |
40 | + *fnamecmpbuf = '\0'; | |
41 | ||
b952a177 WD |
42 | if (statret == 0 && !S_ISREG(st.st_mode)) { |
43 | if (delete_file(fname) != 0) | |
44 | @@ -455,11 +458,9 @@ static void recv_generator(char *fname, | |
77945073 WD |
45 | if (statret == -1) { |
46 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
47 | return; | |
b952a177 | 48 | - if (stat_errno == ENOENT) { |
0bd6d59f | 49 | - write_int(f_out,i); |
2f5fa77e | 50 | - if (!dry_run && !read_batch) |
0bd6d59f WD |
51 | - write_sum_head(f_out, NULL); |
52 | - } else if (verbose > 1) { | |
b952a177 | 53 | + if (stat_errno == ENOENT) |
d24c084d | 54 | + goto notify_others; |
0bd6d59f | 55 | + if (verbose > 1) { |
b952a177 WD |
56 | rsyserr(FERROR, stat_errno, |
57 | "recv_generator: failed to stat %s", | |
0bd6d59f | 58 | full_fname(fname)); |
b952a177 WD |
59 | @@ -467,13 +468,13 @@ static void recv_generator(char *fname, |
60 | return; | |
77945073 WD |
61 | } |
62 | ||
63 | - if (opt_ignore_existing && fnamecmp == fname) { | |
64 | + if (opt_ignore_existing && !*fnamecmpbuf) { | |
65 | if (verbose > 1) | |
b952a177 | 66 | rprintf(FINFO, "%s exists\n", safe_fname(fname)); |
77945073 WD |
67 | return; |
68 | } | |
69 | ||
70 | - if (update_only && fnamecmp == fname | |
71 | + if (update_only && !*fnamecmpbuf | |
72 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { | |
73 | if (verbose > 1) | |
b952a177 WD |
74 | rprintf(FINFO, "%s is newer\n", safe_fname(fname)); |
75 | @@ -481,21 +482,17 @@ static void recv_generator(char *fname, | |
77945073 WD |
76 | } |
77 | ||
78 | if (skip_file(fname, file, &st)) { | |
79 | - if (fnamecmp == fname) | |
80 | + if (!*fnamecmpbuf) | |
81 | set_perms(fname, file, &st, PERMS_REPORT); | |
82 | return; | |
83 | } | |
84 | ||
2f5fa77e | 85 | - if (dry_run || read_batch) { |
0bd6d59f WD |
86 | - write_int(f_out,i); |
87 | - return; | |
88 | - } | |
89 | - | |
90 | - if (whole_file > 0) { | |
91 | - write_int(f_out,i); | |
92 | - write_sum_head(f_out, NULL); | |
93 | - return; | |
94 | + if (dry_run || whole_file) { | |
95 | + statret = -1; | |
d24c084d | 96 | + goto notify_others; |
77945073 | 97 | } |
0bd6d59f | 98 | + if (read_batch) |
d24c084d | 99 | + goto notify_others; |
77945073 | 100 | |
b952a177 WD |
101 | if (partial_dir) { |
102 | STRUCT_STAT st2; | |
103 | @@ -516,9 +513,8 @@ static void recv_generator(char *fname, | |
77945073 WD |
104 | /* pretend the file didn't exist */ |
105 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
106 | return; | |
0bd6d59f WD |
107 | - write_int(f_out,i); |
108 | - write_sum_head(f_out, NULL); | |
109 | - return; | |
110 | + statret = -1; | |
d24c084d | 111 | + goto notify_others; |
0bd6d59f WD |
112 | } |
113 | ||
0bd6d59f | 114 | if (verbose > 3) { |
b952a177 | 115 | @@ -529,14 +525,41 @@ static void recv_generator(char *fname, |
77945073 WD |
116 | if (verbose > 2) |
117 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
118 | ||
0bd6d59f | 119 | - write_int(f_out,i); |
dc3ae351 | 120 | - generate_and_send_sums(fd, st.st_size, f_out); |
d24c084d | 121 | +notify_others: |
d6e18487 | 122 | + if (f_out_name >= 0) { |
0b32df42 | 123 | + uchar lenbuf[3], *lb = lenbuf; |
0bd6d59f | 124 | + int len = statret == -1 ? 0 : strlen(fnamecmpbuf); |
0bd6d59f WD |
125 | + if (len > 0x7F) { |
126 | +#if MAXPATHLEN > 0x7FFF | |
127 | + *lb++ = len / 0x10000 + 0x80; | |
128 | + *lb++ = len / 0x100; | |
0b32df42 | 129 | +#else |
0bd6d59f | 130 | + *lb++ = len / 0x100 + 0x80; |
0b32df42 WD |
131 | +#endif |
132 | + } | |
133 | + *lb = len; | |
d6e18487 | 134 | + write_buf(f_out_name, lenbuf, lb - lenbuf + 1); |
0bd6d59f | 135 | + if (len) |
d6e18487 WD |
136 | + write_buf(f_out_name, fnamecmpbuf, len); |
137 | + io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */ | |
0b32df42 | 138 | + } |
0bd6d59f WD |
139 | + |
140 | + write_int(f_out, i); | |
2f5fa77e | 141 | + |
d6e18487 | 142 | + if (dry_run || read_batch) |
2f5fa77e WD |
143 | + return; |
144 | ||
145 | - close(fd); | |
0bd6d59f | 146 | + if (statret == 0) { |
dc3ae351 | 147 | + generate_and_send_sums(fd, st.st_size, f_out); |
0bd6d59f WD |
148 | + |
149 | + close(fd); | |
d6e18487 | 150 | + } else |
0bd6d59f | 151 | + write_sum_head(f_out, NULL); |
77945073 WD |
152 | } |
153 | ||
154 | ||
9be39c35 WD |
155 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
156 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
d6e18487 | 157 | + int f_out_name) |
77945073 WD |
158 | { |
159 | int i; | |
8cec1ead | 160 | int phase = 0; |
b952a177 | 161 | @@ -577,7 +600,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
162 | } |
163 | ||
164 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 165 | - file, i, f_out); |
d6e18487 | 166 | + file, i, f_out, f_out_name); |
77945073 WD |
167 | } |
168 | ||
169 | phase++; | |
b952a177 | 170 | @@ -594,7 +617,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
171 | while ((i = get_redo_num()) != -1) { |
172 | struct file_struct *file = flist->files[i]; | |
173 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 174 | - file, i, f_out); |
d6e18487 | 175 | + file, i, f_out, f_out_name); |
77945073 WD |
176 | } |
177 | ||
178 | phase++; | |
b952a177 | 179 | @@ -613,7 +636,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
180 | if (!file->basename || !S_ISDIR(file->mode)) |
181 | continue; | |
182 | recv_generator(local_name ? local_name : f_name(file), | |
183 | - file, i, -1); | |
184 | + file, i, -1, -1); | |
185 | } | |
186 | ||
187 | if (verbose > 2) | |
2eb7ca6c | 188 | --- orig/main.c 2004-07-29 16:08:03 |
2f5fa77e WD |
189 | +++ main.c 2004-07-22 00:10:43 |
190 | @@ -58,6 +58,7 @@ extern int filesfrom_fd; | |
0bd6d59f WD |
191 | extern pid_t cleanup_child_pid; |
192 | extern char *files_from; | |
193 | extern char *remote_filesfrom_file; | |
194 | +extern char *compare_dest; | |
195 | extern char *rsync_path; | |
196 | extern char *shell_cmd; | |
197 | extern char *batch_name; | |
d6e18487 | 198 | @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
199 | { |
200 | int pid; | |
8cec1ead | 201 | int status = 0; |
77945073 | 202 | - int error_pipe[2]; |
77945073 | 203 | + int error_pipe[2], name_pipe[2]; |
d6e18487 | 204 | + BOOL need_name_pipe = compare_dest && !dry_run; |
77945073 WD |
205 | |
206 | if (preserve_hard_links) | |
207 | init_hard_links(flist); | |
d6e18487 | 208 | @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st |
d24c084d | 209 | delete_files(flist); |
77945073 WD |
210 | } |
211 | ||
212 | - if (fd_pair(error_pipe) < 0) { | |
213 | - rprintf(FERROR,"error pipe failed in do_recv\n"); | |
0bd6d59f WD |
214 | + if (fd_pair(error_pipe) < 0 |
215 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
77945073 WD |
216 | + rprintf(FERROR, "fd_pair() failed in do_recv\n"); |
217 | exit_cleanup(RERR_SOCKETIO); | |
218 | } | |
219 | ||
d6e18487 | 220 | @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 221 | |
8cec1ead | 222 | if ((pid = do_fork()) == 0) { |
77945073 | 223 | close(error_pipe[0]); |
0bd6d59f WD |
224 | + if (need_name_pipe) { |
225 | + close(name_pipe[1]); | |
226 | + set_blocking(name_pipe[0]); | |
227 | + } else | |
228 | + name_pipe[0] = -1; | |
77945073 WD |
229 | if (f_in != f_out) |
230 | close(f_out); | |
77945073 | 231 | |
d6e18487 | 232 | @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
233 | /* set place to send errors */ |
234 | set_msg_fd_out(error_pipe[1]); | |
235 | ||
236 | - recv_files(f_in,flist,local_name); | |
237 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
238 | io_flush(FULL_FLUSH); | |
239 | report(f_in); | |
240 | ||
2eb7ca6c | 241 | @@ -504,6 +512,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 242 | stop_write_batch(); |
77945073 WD |
243 | |
244 | close(error_pipe[1]); | |
0bd6d59f WD |
245 | + if (need_name_pipe) { |
246 | + close(name_pipe[0]); | |
247 | + set_nonblocking(name_pipe[1]); | |
248 | + } else | |
249 | + name_pipe[1] = -1; | |
77945073 WD |
250 | if (f_in != f_out) |
251 | close(f_in); | |
77945073 | 252 | |
2eb7ca6c | 253 | @@ -511,7 +524,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
254 | |
255 | set_msg_fd_in(error_pipe[0]); | |
256 | ||
257 | - generate_files(f_out, flist, local_name); | |
258 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
259 | ||
260 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
261 | report(-1); | |
2eb7ca6c | 262 | --- orig/receiver.c 2004-07-29 16:08:03 |
b952a177 | 263 | +++ receiver.c 2004-07-23 21:59:07 |
2eb7ca6c | 264 | @@ -323,6 +323,30 @@ static int receive_data(int f_in, char * |
0bd6d59f WD |
265 | } |
266 | ||
5823d322 | 267 | |
0bd6d59f WD |
268 | +static char *read_gen_name(int fd, char *buf, char *realname) |
269 | +{ | |
270 | + int len = read_byte(fd); | |
271 | + if (len & 0x80) { | |
272 | +#if MAXPATHLEN > 32767 | |
273 | + uchar lenbuf[2]; | |
274 | + read_buf(fd, (char *)lenbuf, 2); | |
275 | + len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; | |
276 | +#else | |
277 | + len = (len & ~0x80) * 0x100 + read_byte(fd); | |
278 | +#endif | |
279 | + } | |
280 | + if (len) { | |
281 | + if (len >= MAXPATHLEN) { | |
282 | + rprintf(FERROR, "bogus data on generator name pipe\n"); | |
283 | + exit_cleanup(RERR_PROTOCOL); | |
284 | + } | |
285 | + read_sbuf(fd, buf, len); | |
286 | + return buf; | |
287 | + } | |
288 | + return realname; | |
289 | +} | |
290 | + | |
5823d322 WD |
291 | + |
292 | static void discard_receive_data(int f_in, OFF_T length) | |
293 | { | |
dc3ae351 | 294 | receive_data(f_in, NULL, -1, 0, NULL, -1, length); |
2eb7ca6c | 295 | @@ -333,7 +357,8 @@ static void discard_receive_data(int f_i |
77945073 WD |
296 | * main routine for receiver process. |
297 | * | |
298 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 299 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 300 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
d6e18487 | 301 | + int f_in_name) |
77945073 | 302 | { |
2f5fa77e | 303 | int next_gen_i = -1; |
77945073 | 304 | int fd1,fd2; |
2eb7ca6c | 305 | @@ -362,8 +387,15 @@ int recv_files(int f_in, struct file_lis |
0bd6d59f WD |
306 | i = read_int(f_in); |
307 | if (i == -1) { | |
2f5fa77e WD |
308 | if (read_batch) { |
309 | - if (next_gen_i != flist->count) | |
310 | - while (read_int(batch_gen_fd) != -1) {} | |
0bd6d59f | 311 | + while (next_gen_i < flist->count) { |
d6e18487 WD |
312 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
313 | + read_gen_name(f_in_name, | |
2f5fa77e WD |
314 | + fnamecmpbuf, NULL); |
315 | + } | |
d6e18487 | 316 | + next_gen_i = read_int(batch_gen_fd); |
2f5fa77e WD |
317 | + if (next_gen_i < 0) |
318 | + break; | |
0bd6d59f | 319 | + } |
2f5fa77e WD |
320 | next_gen_i = -1; |
321 | } | |
5823d322 | 322 | |
2eb7ca6c | 323 | @@ -411,6 +443,10 @@ int recv_files(int f_in, struct file_lis |
77945073 | 324 | |
2f5fa77e WD |
325 | if (read_batch) { |
326 | while (i > next_gen_i) { | |
d6e18487 WD |
327 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
328 | + read_gen_name(f_in_name, fnamecmpbuf, | |
2f5fa77e WD |
329 | + NULL); |
330 | + } | |
331 | next_gen_i = read_int(batch_gen_fd); | |
332 | if (next_gen_i == -1) | |
333 | next_gen_i = flist->count; | |
2eb7ca6c | 334 | @@ -421,8 +457,14 @@ int recv_files(int f_in, struct file_lis |
2f5fa77e WD |
335 | discard_receive_data(f_in, file->length); |
336 | continue; | |
337 | } | |
0bd6d59f | 338 | + next_gen_i = -1; |
2f5fa77e WD |
339 | } |
340 | ||
d6e18487 WD |
341 | + if (f_in_name >= 0) |
342 | + fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname); | |
0bd6d59f | 343 | + else |
0b32df42 | 344 | + fnamecmp = fname; |
2f5fa77e | 345 | + |
5823d322 WD |
346 | if (server_exclude_list.head |
347 | && check_exclude(&server_exclude_list, fname, | |
2f5fa77e | 348 | S_ISDIR(file->mode)) < 0) { |
2eb7ca6c | 349 | @@ -435,13 +477,7 @@ int recv_files(int f_in, struct file_lis |
b952a177 WD |
350 | continue; |
351 | } | |
352 | ||
353 | - if (partial_dir) { | |
354 | - if ((partialptr = partial_dir_fname(fname)) != NULL) | |
355 | - fnamecmp = partialptr; | |
356 | - else | |
357 | - fnamecmp = fname; | |
358 | - } else | |
359 | - fnamecmp = partialptr = fname; | |
360 | + partialptr = partial_dir ? partial_dir_fname(fname) : fname; | |
361 | ||
77945073 WD |
362 | /* open the file */ |
363 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
2eb7ca6c | 364 | @@ -451,14 +487,6 @@ int recv_files(int f_in, struct file_lis |
b952a177 WD |
365 | fd1 = do_open(fnamecmp, O_RDONLY, 0); |
366 | } | |
77945073 WD |
367 | |
368 | - if (fd1 == -1 && compare_dest != NULL) { | |
369 | - /* try the file at compare_dest instead */ | |
370 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
371 | - compare_dest, fname); | |
372 | - fnamecmp = fnamecmpbuf; | |
373 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
374 | - } | |
375 | - | |
376 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
377 | rsyserr(FERROR, errno, "fstat %s failed", | |
378 | full_fname(fnamecmp)); |