Commit | Line | Data |
---|---|---|
b952a177 WD |
1 | Depends-On-Patch: partial-dir.diff |
2 | ||
77945073 WD |
3 | Added a pipe from the generator to the receiver that communicates |
4 | what basis file we used to generate the file data (if it was not | |
5 | the default name). This optimizes away the basis-file search in | |
6 | the receiver and makes future options that do more basis-file | |
7 | searching more efficient (such as the --fuzzy option and the | |
8 | support for multiple --compare-dest options). | |
9 | ||
10 | You must run "make proto" before compiling. | |
11 | ||
b952a177 WD |
12 | --- orig/generator.c 2004-07-28 09:58:28 |
13 | +++ generator.c 2004-07-28 10:14:15 | |
14 | @@ -259,9 +259,9 @@ static void generate_and_send_sums(int f | |
77945073 WD |
15 | * out. It might be wrong. |
16 | */ | |
17 | static void recv_generator(char *fname, struct file_struct *file, int i, | |
18 | - int f_out) | |
d6e18487 | 19 | + int f_out, int f_out_name) |
77945073 | 20 | { |
0bd6d59f WD |
21 | - int fd; |
22 | + int fd = -1; | |
77945073 | 23 | STRUCT_STAT st; |
b952a177 | 24 | int statret, stat_errno; |
0bd6d59f | 25 | char *fnamecmp; |
b952a177 WD |
26 | @@ -437,13 +437,16 @@ static void recv_generator(char *fname, |
27 | safe_fname(fname)); | |
28 | } | |
29 | fnamecmp = fnamecmpbuf; | |
30 | - } | |
31 | + } else | |
32 | + *fnamecmpbuf = '\0'; | |
33 | } else | |
77945073 | 34 | #endif |
b952a177 WD |
35 | fnamecmp = fnamecmpbuf; |
36 | statret = 0; | |
37 | - } | |
77945073 | 38 | - } |
b952a177 WD |
39 | + } else |
40 | + *fnamecmpbuf = '\0'; | |
77945073 WD |
41 | + } else |
42 | + *fnamecmpbuf = '\0'; | |
43 | ||
b952a177 WD |
44 | if (statret == 0 && !S_ISREG(st.st_mode)) { |
45 | if (delete_file(fname) != 0) | |
46 | @@ -455,11 +458,9 @@ static void recv_generator(char *fname, | |
77945073 WD |
47 | if (statret == -1) { |
48 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
49 | return; | |
b952a177 | 50 | - if (stat_errno == ENOENT) { |
0bd6d59f | 51 | - write_int(f_out,i); |
2f5fa77e | 52 | - if (!dry_run && !read_batch) |
0bd6d59f WD |
53 | - write_sum_head(f_out, NULL); |
54 | - } else if (verbose > 1) { | |
b952a177 | 55 | + if (stat_errno == ENOENT) |
d24c084d | 56 | + goto notify_others; |
0bd6d59f | 57 | + if (verbose > 1) { |
b952a177 WD |
58 | rsyserr(FERROR, stat_errno, |
59 | "recv_generator: failed to stat %s", | |
0bd6d59f | 60 | full_fname(fname)); |
b952a177 WD |
61 | @@ -467,13 +468,13 @@ static void recv_generator(char *fname, |
62 | return; | |
77945073 WD |
63 | } |
64 | ||
65 | - if (opt_ignore_existing && fnamecmp == fname) { | |
66 | + if (opt_ignore_existing && !*fnamecmpbuf) { | |
67 | if (verbose > 1) | |
b952a177 | 68 | rprintf(FINFO, "%s exists\n", safe_fname(fname)); |
77945073 WD |
69 | return; |
70 | } | |
71 | ||
72 | - if (update_only && fnamecmp == fname | |
73 | + if (update_only && !*fnamecmpbuf | |
74 | && cmp_modtime(st.st_mtime, file->modtime) > 0) { | |
75 | if (verbose > 1) | |
b952a177 WD |
76 | rprintf(FINFO, "%s is newer\n", safe_fname(fname)); |
77 | @@ -481,21 +482,17 @@ static void recv_generator(char *fname, | |
77945073 WD |
78 | } |
79 | ||
80 | if (skip_file(fname, file, &st)) { | |
81 | - if (fnamecmp == fname) | |
82 | + if (!*fnamecmpbuf) | |
83 | set_perms(fname, file, &st, PERMS_REPORT); | |
84 | return; | |
85 | } | |
86 | ||
2f5fa77e | 87 | - if (dry_run || read_batch) { |
0bd6d59f WD |
88 | - write_int(f_out,i); |
89 | - return; | |
90 | - } | |
91 | - | |
92 | - if (whole_file > 0) { | |
93 | - write_int(f_out,i); | |
94 | - write_sum_head(f_out, NULL); | |
95 | - return; | |
96 | + if (dry_run || whole_file) { | |
97 | + statret = -1; | |
d24c084d | 98 | + goto notify_others; |
77945073 | 99 | } |
0bd6d59f | 100 | + if (read_batch) |
d24c084d | 101 | + goto notify_others; |
77945073 | 102 | |
b952a177 WD |
103 | if (partial_dir) { |
104 | STRUCT_STAT st2; | |
105 | @@ -516,9 +513,8 @@ static void recv_generator(char *fname, | |
77945073 WD |
106 | /* pretend the file didn't exist */ |
107 | if (preserve_hard_links && hard_link_check(file, HL_SKIP)) | |
108 | return; | |
0bd6d59f WD |
109 | - write_int(f_out,i); |
110 | - write_sum_head(f_out, NULL); | |
111 | - return; | |
112 | + statret = -1; | |
d24c084d | 113 | + goto notify_others; |
0bd6d59f WD |
114 | } |
115 | ||
0bd6d59f | 116 | if (verbose > 3) { |
b952a177 | 117 | @@ -529,14 +525,41 @@ static void recv_generator(char *fname, |
77945073 WD |
118 | if (verbose > 2) |
119 | rprintf(FINFO, "generating and sending sums for %d\n", i); | |
120 | ||
0bd6d59f | 121 | - write_int(f_out,i); |
dc3ae351 | 122 | - generate_and_send_sums(fd, st.st_size, f_out); |
d24c084d | 123 | +notify_others: |
d6e18487 | 124 | + if (f_out_name >= 0) { |
0b32df42 | 125 | + uchar lenbuf[3], *lb = lenbuf; |
0bd6d59f | 126 | + int len = statret == -1 ? 0 : strlen(fnamecmpbuf); |
0bd6d59f WD |
127 | + if (len > 0x7F) { |
128 | +#if MAXPATHLEN > 0x7FFF | |
129 | + *lb++ = len / 0x10000 + 0x80; | |
130 | + *lb++ = len / 0x100; | |
0b32df42 | 131 | +#else |
0bd6d59f | 132 | + *lb++ = len / 0x100 + 0x80; |
0b32df42 WD |
133 | +#endif |
134 | + } | |
135 | + *lb = len; | |
d6e18487 | 136 | + write_buf(f_out_name, lenbuf, lb - lenbuf + 1); |
0bd6d59f | 137 | + if (len) |
d6e18487 WD |
138 | + write_buf(f_out_name, fnamecmpbuf, len); |
139 | + io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */ | |
0b32df42 | 140 | + } |
0bd6d59f WD |
141 | + |
142 | + write_int(f_out, i); | |
2f5fa77e | 143 | + |
d6e18487 | 144 | + if (dry_run || read_batch) |
2f5fa77e WD |
145 | + return; |
146 | ||
147 | - close(fd); | |
0bd6d59f | 148 | + if (statret == 0) { |
dc3ae351 | 149 | + generate_and_send_sums(fd, st.st_size, f_out); |
0bd6d59f WD |
150 | + |
151 | + close(fd); | |
d6e18487 | 152 | + } else |
0bd6d59f | 153 | + write_sum_head(f_out, NULL); |
77945073 WD |
154 | } |
155 | ||
156 | ||
9be39c35 WD |
157 | -void generate_files(int f_out, struct file_list *flist, char *local_name) |
158 | +void generate_files(int f_out, struct file_list *flist, char *local_name, | |
d6e18487 | 159 | + int f_out_name) |
77945073 WD |
160 | { |
161 | int i; | |
8cec1ead | 162 | int phase = 0; |
b952a177 | 163 | @@ -577,7 +600,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
164 | } |
165 | ||
166 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 167 | - file, i, f_out); |
d6e18487 | 168 | + file, i, f_out, f_out_name); |
77945073 WD |
169 | } |
170 | ||
171 | phase++; | |
b952a177 | 172 | @@ -594,7 +617,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
173 | while ((i = get_redo_num()) != -1) { |
174 | struct file_struct *file = flist->files[i]; | |
175 | recv_generator(local_name ? local_name : f_name_to(file, fbuf), | |
9be39c35 | 176 | - file, i, f_out); |
d6e18487 | 177 | + file, i, f_out, f_out_name); |
77945073 WD |
178 | } |
179 | ||
180 | phase++; | |
b952a177 | 181 | @@ -613,7 +636,7 @@ void generate_files(int f_out, struct fi |
77945073 WD |
182 | if (!file->basename || !S_ISDIR(file->mode)) |
183 | continue; | |
184 | recv_generator(local_name ? local_name : f_name(file), | |
185 | - file, i, -1); | |
186 | + file, i, -1, -1); | |
187 | } | |
188 | ||
189 | if (verbose > 2) | |
b952a177 | 190 | --- orig/main.c 2004-07-24 16:52:09 |
2f5fa77e WD |
191 | +++ main.c 2004-07-22 00:10:43 |
192 | @@ -58,6 +58,7 @@ extern int filesfrom_fd; | |
0bd6d59f WD |
193 | extern pid_t cleanup_child_pid; |
194 | extern char *files_from; | |
195 | extern char *remote_filesfrom_file; | |
196 | +extern char *compare_dest; | |
197 | extern char *rsync_path; | |
198 | extern char *shell_cmd; | |
199 | extern char *batch_name; | |
d6e18487 | 200 | @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
201 | { |
202 | int pid; | |
8cec1ead | 203 | int status = 0; |
77945073 | 204 | - int error_pipe[2]; |
77945073 | 205 | + int error_pipe[2], name_pipe[2]; |
d6e18487 | 206 | + BOOL need_name_pipe = compare_dest && !dry_run; |
77945073 WD |
207 | |
208 | if (preserve_hard_links) | |
209 | init_hard_links(flist); | |
d6e18487 | 210 | @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st |
d24c084d | 211 | delete_files(flist); |
77945073 WD |
212 | } |
213 | ||
214 | - if (fd_pair(error_pipe) < 0) { | |
215 | - rprintf(FERROR,"error pipe failed in do_recv\n"); | |
0bd6d59f WD |
216 | + if (fd_pair(error_pipe) < 0 |
217 | + || (need_name_pipe && fd_pair(name_pipe) < 0)) { | |
77945073 WD |
218 | + rprintf(FERROR, "fd_pair() failed in do_recv\n"); |
219 | exit_cleanup(RERR_SOCKETIO); | |
220 | } | |
221 | ||
d6e18487 | 222 | @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st |
77945073 | 223 | |
8cec1ead | 224 | if ((pid = do_fork()) == 0) { |
77945073 | 225 | close(error_pipe[0]); |
0bd6d59f WD |
226 | + if (need_name_pipe) { |
227 | + close(name_pipe[1]); | |
228 | + set_blocking(name_pipe[0]); | |
229 | + } else | |
230 | + name_pipe[0] = -1; | |
77945073 WD |
231 | if (f_in != f_out) |
232 | close(f_out); | |
77945073 | 233 | |
d6e18487 | 234 | @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
235 | /* set place to send errors */ |
236 | set_msg_fd_out(error_pipe[1]); | |
237 | ||
238 | - recv_files(f_in,flist,local_name); | |
239 | + recv_files(f_in, flist, local_name, name_pipe[0]); | |
240 | io_flush(FULL_FLUSH); | |
241 | report(f_in); | |
242 | ||
d6e18487 | 243 | @@ -503,6 +511,11 @@ static int do_recv(int f_in,int f_out,st |
9be39c35 | 244 | stop_write_batch(); |
77945073 WD |
245 | |
246 | close(error_pipe[1]); | |
0bd6d59f WD |
247 | + if (need_name_pipe) { |
248 | + close(name_pipe[0]); | |
249 | + set_nonblocking(name_pipe[1]); | |
250 | + } else | |
251 | + name_pipe[1] = -1; | |
77945073 WD |
252 | if (f_in != f_out) |
253 | close(f_in); | |
77945073 | 254 | |
d6e18487 | 255 | @@ -510,7 +523,7 @@ static int do_recv(int f_in,int f_out,st |
77945073 WD |
256 | |
257 | set_msg_fd_in(error_pipe[0]); | |
258 | ||
259 | - generate_files(f_out, flist, local_name); | |
260 | + generate_files(f_out, flist, local_name, name_pipe[1]); | |
261 | ||
262 | get_redo_num(); /* Read final MSG_DONE and any prior messages. */ | |
263 | report(-1); | |
b952a177 WD |
264 | --- orig/receiver.c 2004-07-27 23:26:20 |
265 | +++ receiver.c 2004-07-23 21:59:07 | |
266 | @@ -325,6 +325,30 @@ static int receive_data(int f_in, char * | |
0bd6d59f WD |
267 | } |
268 | ||
5823d322 | 269 | |
0bd6d59f WD |
270 | +static char *read_gen_name(int fd, char *buf, char *realname) |
271 | +{ | |
272 | + int len = read_byte(fd); | |
273 | + if (len & 0x80) { | |
274 | +#if MAXPATHLEN > 32767 | |
275 | + uchar lenbuf[2]; | |
276 | + read_buf(fd, (char *)lenbuf, 2); | |
277 | + len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; | |
278 | +#else | |
279 | + len = (len & ~0x80) * 0x100 + read_byte(fd); | |
280 | +#endif | |
281 | + } | |
282 | + if (len) { | |
283 | + if (len >= MAXPATHLEN) { | |
284 | + rprintf(FERROR, "bogus data on generator name pipe\n"); | |
285 | + exit_cleanup(RERR_PROTOCOL); | |
286 | + } | |
287 | + read_sbuf(fd, buf, len); | |
288 | + return buf; | |
289 | + } | |
290 | + return realname; | |
291 | +} | |
292 | + | |
5823d322 WD |
293 | + |
294 | static void discard_receive_data(int f_in, OFF_T length) | |
295 | { | |
dc3ae351 | 296 | receive_data(f_in, NULL, -1, 0, NULL, -1, length); |
b952a177 | 297 | @@ -335,7 +359,8 @@ static void discard_receive_data(int f_i |
77945073 WD |
298 | * main routine for receiver process. |
299 | * | |
300 | * Receiver process runs on the same host as the generator process. */ | |
9be39c35 | 301 | -int recv_files(int f_in, struct file_list *flist, char *local_name) |
77945073 | 302 | +int recv_files(int f_in, struct file_list *flist, char *local_name, |
d6e18487 | 303 | + int f_in_name) |
77945073 | 304 | { |
2f5fa77e | 305 | int next_gen_i = -1; |
77945073 | 306 | int fd1,fd2; |
b952a177 | 307 | @@ -364,8 +389,15 @@ int recv_files(int f_in, struct file_lis |
0bd6d59f WD |
308 | i = read_int(f_in); |
309 | if (i == -1) { | |
2f5fa77e WD |
310 | if (read_batch) { |
311 | - if (next_gen_i != flist->count) | |
312 | - while (read_int(batch_gen_fd) != -1) {} | |
0bd6d59f | 313 | + while (next_gen_i < flist->count) { |
d6e18487 WD |
314 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
315 | + read_gen_name(f_in_name, | |
2f5fa77e WD |
316 | + fnamecmpbuf, NULL); |
317 | + } | |
d6e18487 | 318 | + next_gen_i = read_int(batch_gen_fd); |
2f5fa77e WD |
319 | + if (next_gen_i < 0) |
320 | + break; | |
0bd6d59f | 321 | + } |
2f5fa77e WD |
322 | next_gen_i = -1; |
323 | } | |
5823d322 | 324 | |
b952a177 | 325 | @@ -413,6 +445,10 @@ int recv_files(int f_in, struct file_lis |
77945073 | 326 | |
2f5fa77e WD |
327 | if (read_batch) { |
328 | while (i > next_gen_i) { | |
d6e18487 WD |
329 | + if (f_in_name >= 0 && next_gen_i >= 0) { |
330 | + read_gen_name(f_in_name, fnamecmpbuf, | |
2f5fa77e WD |
331 | + NULL); |
332 | + } | |
333 | next_gen_i = read_int(batch_gen_fd); | |
334 | if (next_gen_i == -1) | |
335 | next_gen_i = flist->count; | |
b952a177 | 336 | @@ -423,8 +459,14 @@ int recv_files(int f_in, struct file_lis |
2f5fa77e WD |
337 | discard_receive_data(f_in, file->length); |
338 | continue; | |
339 | } | |
0bd6d59f | 340 | + next_gen_i = -1; |
2f5fa77e WD |
341 | } |
342 | ||
d6e18487 WD |
343 | + if (f_in_name >= 0) |
344 | + fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname); | |
0bd6d59f | 345 | + else |
0b32df42 | 346 | + fnamecmp = fname; |
2f5fa77e | 347 | + |
5823d322 WD |
348 | if (server_exclude_list.head |
349 | && check_exclude(&server_exclude_list, fname, | |
2f5fa77e | 350 | S_ISDIR(file->mode)) < 0) { |
b952a177 WD |
351 | @@ -437,13 +479,7 @@ int recv_files(int f_in, struct file_lis |
352 | continue; | |
353 | } | |
354 | ||
355 | - if (partial_dir) { | |
356 | - if ((partialptr = partial_dir_fname(fname)) != NULL) | |
357 | - fnamecmp = partialptr; | |
358 | - else | |
359 | - fnamecmp = fname; | |
360 | - } else | |
361 | - fnamecmp = partialptr = fname; | |
362 | + partialptr = partial_dir ? partial_dir_fname(fname) : fname; | |
363 | ||
77945073 WD |
364 | /* open the file */ |
365 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
b952a177 WD |
366 | @@ -453,14 +489,6 @@ int recv_files(int f_in, struct file_lis |
367 | fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
368 | } | |
77945073 WD |
369 | |
370 | - if (fd1 == -1 && compare_dest != NULL) { | |
371 | - /* try the file at compare_dest instead */ | |
372 | - pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, | |
373 | - compare_dest, fname); | |
374 | - fnamecmp = fnamecmpbuf; | |
375 | - fd1 = do_open(fnamecmp, O_RDONLY, 0); | |
376 | - } | |
377 | - | |
378 | if (fd1 != -1 && do_fstat(fd1,&st) != 0) { | |
379 | rsyserr(FERROR, errno, "fstat %s failed", | |
380 | full_fname(fnamecmp)); |