Tweaked a label name.
[rsync/rsync-patches.git] / g2r-basis-filename.diff
CommitLineData
77945073
WD
1Added a pipe from the generator to the receiver that communicates
2what basis file we used to generate the file data (if it was not
3the default name). This optimizes away the basis-file search in
4the receiver and makes future options that do more basis-file
5searching more efficient (such as the --fuzzy option and the
6support for multiple --compare-dest options).
7
0bd6d59f
WD
8Also fixes a potential synchronization problem between the generator
9and the receiver in read-batch mode. Should consider making the
10sending of the index value the default for this pipe (it's currently
11only sent in batch mode due to the sender not listening to the
12generator to determine what files get updated).
13
77945073
WD
14You must run "make proto" before compiling.
15
0bd6d59f
WD
16--- orig/generator.c 2004-07-17 16:30:20
17+++ generator.c 2004-07-17 15:50:09
18@@ -251,11 +251,11 @@ static void generate_and_send_sums(struc
77945073
WD
19 * out. It might be wrong.
20 */
21 static void recv_generator(char *fname, struct file_struct *file, int i,
22- int f_out)
23+ int f_out, int f_nameout)
24 {
0bd6d59f
WD
25- int fd;
26+ int fd = -1;
77945073 27 STRUCT_STAT st;
0bd6d59f
WD
28- struct map_struct *mapbuf;
29+ struct map_struct *mapbuf = NULL;
30 int statret;
31 char *fnamecmp;
32 char fnamecmpbuf[MAXPATHLEN];
33@@ -398,9 +398,6 @@ static void recv_generator(char *fname,
34 }
35 #endif
36
37- if (read_batch)
38- return;
39-
40 if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
41 return;
42
43@@ -418,8 +415,10 @@ static void recv_generator(char *fname,
77945073
WD
44 statret = link_stat(fnamecmpbuf, &st, 0);
45 if (!S_ISREG(st.st_mode))
46 statret = -1;
47- if (statret == -1)
0bd6d59f 48+ if (statret == -1) {
77945073
WD
49 errno = saveerrno;
50+ *fnamecmpbuf = '\0';
51+ }
52 #if HAVE_LINK
53 else if (link_dest && !dry_run) {
54 if (do_link(fnamecmpbuf, fname) != 0) {
0bd6d59f 55@@ -427,22 +426,22 @@ static void recv_generator(char *fname,
77945073
WD
56 rsyserr(FINFO, errno, "link %s => %s",
57 fnamecmpbuf, fname);
58 }
59- }
60- fnamecmp = fnamecmpbuf;
61+ fnamecmp = fnamecmpbuf;
62+ } else
63+ *fnamecmpbuf = '\0';
64 }
65 #endif
66 else
67 fnamecmp = fnamecmpbuf;
68- }
69+ } else
70+ *fnamecmpbuf = '\0';
71
72 if (statret == -1) {
73 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
74 return;
0bd6d59f
WD
75- if (errno == ENOENT) {
76- write_int(f_out,i);
77- if (!dry_run)
78- write_sum_head(f_out, NULL);
79- } else if (verbose > 1) {
80+ if (errno == ENOENT)
d24c084d 81+ goto notify_others;
0bd6d59f
WD
82+ if (verbose > 1) {
83 rsyserr(FERROR, errno,
84 "recv_generator: failed to open %s",
85 full_fname(fname));
86@@ -451,26 +450,23 @@ static void recv_generator(char *fname,
87 }
88
89 if (!S_ISREG(st.st_mode)) {
90- if (delete_file(fname) != 0) {
91+ if (delete_file(fname) != 0)
92 return;
93- }
94
77945073
WD
95 /* now pretend the file didn't exist */
96 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
97 return;
0bd6d59f
WD
98- write_int(f_out,i);
99- if (!dry_run)
100- write_sum_head(f_out, NULL);
101- return;
102+ statret = -1;
d24c084d 103+ goto notify_others;
77945073
WD
104 }
105
106- if (opt_ignore_existing && fnamecmp == fname) {
107+ if (opt_ignore_existing && !*fnamecmpbuf) {
108 if (verbose > 1)
109 rprintf(FINFO,"%s exists\n",fname);
110 return;
111 }
112
113- if (update_only && fnamecmp == fname
114+ if (update_only && !*fnamecmpbuf
115 && cmp_modtime(st.st_mtime, file->modtime) > 0) {
116 if (verbose > 1)
117 rprintf(FINFO,"%s is newer\n",fname);
0bd6d59f 118@@ -478,21 +474,17 @@ static void recv_generator(char *fname,
77945073
WD
119 }
120
121 if (skip_file(fname, file, &st)) {
122- if (fnamecmp == fname)
123+ if (!*fnamecmpbuf)
124 set_perms(fname, file, &st, PERMS_REPORT);
125 return;
126 }
127
0bd6d59f
WD
128- if (dry_run) {
129- write_int(f_out,i);
130- return;
131- }
132-
133- if (whole_file > 0) {
134- write_int(f_out,i);
135- write_sum_head(f_out, NULL);
136- return;
137+ if (dry_run || whole_file) {
138+ statret = -1;
d24c084d 139+ goto notify_others;
77945073 140 }
0bd6d59f 141+ if (read_batch)
d24c084d 142+ goto notify_others;
77945073 143
0bd6d59f
WD
144 /* open the file */
145 fd = do_open(fnamecmp, O_RDONLY, 0);
146@@ -503,15 +495,12 @@ static void recv_generator(char *fname,
77945073
WD
147 /* pretend the file didn't exist */
148 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
149 return;
0bd6d59f
WD
150- write_int(f_out,i);
151- write_sum_head(f_out, NULL);
152- return;
153+ statret = -1;
d24c084d 154+ goto notify_others;
0bd6d59f
WD
155 }
156
157 if (st.st_size > 0)
158 mapbuf = map_file(fd,st.st_size);
159- else
160- mapbuf = NULL;
161
162 if (verbose > 3) {
163 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
164@@ -521,16 +510,43 @@ static void recv_generator(char *fname,
77945073
WD
165 if (verbose > 2)
166 rprintf(FINFO, "generating and sending sums for %d\n", i);
167
0bd6d59f
WD
168- write_int(f_out,i);
169- generate_and_send_sums(mapbuf, st.st_size, f_out);
d24c084d 170+notify_others:
0b32df42
WD
171+ if (f_nameout >= 0) {
172+ uchar lenbuf[3], *lb = lenbuf;
0bd6d59f
WD
173+ int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
174+ if (read_batch)
175+ write_int(f_nameout, i);
176+ if (len > 0x7F) {
177+#if MAXPATHLEN > 0x7FFF
178+ *lb++ = len / 0x10000 + 0x80;
179+ *lb++ = len / 0x100;
0b32df42 180+#else
0bd6d59f 181+ *lb++ = len / 0x100 + 0x80;
0b32df42
WD
182+#endif
183+ }
184+ *lb = len;
185+ write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
0bd6d59f
WD
186+ if (len)
187+ write_buf(f_nameout, fnamecmpbuf, len);
0b32df42 188+ }
77945073 189
0bd6d59f
WD
190- close(fd);
191- if (mapbuf)
192- unmap_file(mapbuf);
193+ if (read_batch)
194+ return;
195+
196+ write_int(f_out, i);
197+ if (statret == 0) {
198+ generate_and_send_sums(mapbuf, st.st_size, f_out);
199+
200+ close(fd);
201+ if (mapbuf)
202+ unmap_file(mapbuf);
203+ } else if (!dry_run)
204+ write_sum_head(f_out, NULL);
77945073
WD
205 }
206
207
9be39c35
WD
208-void generate_files(int f_out, struct file_list *flist, char *local_name)
209+void generate_files(int f_out, struct file_list *flist, char *local_name,
77945073
WD
210+ int f_nameout)
211 {
212 int i;
8cec1ead 213 int phase = 0;
0bd6d59f 214@@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi
77945073
WD
215 }
216
217 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35
WD
218- file, i, f_out);
219+ file, i, f_out, f_nameout);
77945073
WD
220 }
221
222 phase++;
0bd6d59f
WD
223@@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi
224 rprintf(FINFO,"generate_files phase=%d\n",phase);
225
226 write_int(f_out, -1);
227+ if (read_batch)
228+ write_int(f_nameout, flist->count);
229
230 /* files can cycle through the system more than once
231 * to catch initial checksum errors */
77945073
WD
232 while ((i = get_redo_num()) != -1) {
233 struct file_struct *file = flist->files[i];
234 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35
WD
235- file, i, f_out);
236+ file, i, f_out, f_nameout);
77945073
WD
237 }
238
239 phase++;
0bd6d59f
WD
240@@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi
241 rprintf(FINFO,"generate_files phase=%d\n",phase);
242
243 write_int(f_out, -1);
244+ if (read_batch)
245+ write_int(f_nameout, flist->count);
246
247 if (preserve_hard_links)
248 do_hard_links();
249@@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi
77945073
WD
250 if (!file->basename || !S_ISDIR(file->mode))
251 continue;
252 recv_generator(local_name ? local_name : f_name(file),
253- file, i, -1);
254+ file, i, -1, -1);
255 }
256
257 if (verbose > 2)
0b32df42 258--- orig/main.c 2004-07-17 15:20:05
0bd6d59f
WD
259+++ main.c 2004-07-17 15:58:11
260@@ -57,6 +57,7 @@ extern int filesfrom_fd;
261 extern pid_t cleanup_child_pid;
262 extern char *files_from;
263 extern char *remote_filesfrom_file;
264+extern char *compare_dest;
265 extern char *rsync_path;
266 extern char *shell_cmd;
267 extern char *batch_name;
d24c084d 268@@ -444,20 +445,21 @@ static int do_recv(int f_in,int f_out,st
77945073
WD
269 {
270 int pid;
8cec1ead 271 int status = 0;
77945073 272- int error_pipe[2];
77945073 273+ int error_pipe[2], name_pipe[2];
0bd6d59f 274+ int need_name_pipe = compare_dest || read_batch;
77945073
WD
275
276 if (preserve_hard_links)
277 init_hard_links(flist);
d24c084d
WD
278
279 if (!delete_after) {
280 /* I moved this here from recv_files() to prevent a race condition */
281- if (recurse && delete_mode && !local_name && flist->count>0) {
282+ if (recurse && delete_mode && !local_name && flist->count > 0)
283 delete_files(flist);
284- }
77945073
WD
285 }
286
287- if (fd_pair(error_pipe) < 0) {
288- rprintf(FERROR,"error pipe failed in do_recv\n");
0bd6d59f
WD
289+ if (fd_pair(error_pipe) < 0
290+ || (need_name_pipe && fd_pair(name_pipe) < 0)) {
77945073
WD
291+ rprintf(FERROR, "fd_pair() failed in do_recv\n");
292 exit_cleanup(RERR_SOCKETIO);
293 }
294
d24c084d 295@@ -465,6 +467,11 @@ static int do_recv(int f_in,int f_out,st
77945073 296
8cec1ead 297 if ((pid = do_fork()) == 0) {
77945073 298 close(error_pipe[0]);
0bd6d59f
WD
299+ if (need_name_pipe) {
300+ close(name_pipe[1]);
301+ set_blocking(name_pipe[0]);
302+ } else
303+ name_pipe[0] = -1;
77945073
WD
304 if (f_in != f_out)
305 close(f_out);
77945073 306
d24c084d 307@@ -474,7 +481,7 @@ static int do_recv(int f_in,int f_out,st
77945073
WD
308 /* set place to send errors */
309 set_msg_fd_out(error_pipe[1]);
310
311- recv_files(f_in,flist,local_name);
312+ recv_files(f_in, flist, local_name, name_pipe[0]);
313 io_flush(FULL_FLUSH);
314 report(f_in);
315
d24c084d 316@@ -492,6 +499,11 @@ static int do_recv(int f_in,int f_out,st
9be39c35 317 stop_write_batch();
77945073
WD
318
319 close(error_pipe[1]);
0bd6d59f
WD
320+ if (need_name_pipe) {
321+ close(name_pipe[0]);
322+ set_nonblocking(name_pipe[1]);
323+ } else
324+ name_pipe[1] = -1;
77945073
WD
325 if (f_in != f_out)
326 close(f_in);
77945073 327
d24c084d 328@@ -499,7 +511,7 @@ static int do_recv(int f_in,int f_out,st
77945073
WD
329
330 set_msg_fd_in(error_pipe[0]);
331
332- generate_files(f_out, flist, local_name);
333+ generate_files(f_out, flist, local_name, name_pipe[1]);
334
335 get_redo_num(); /* Read final MSG_DONE and any prior messages. */
336 report(-1);
f6c3b300 337--- orig/receiver.c 2004-07-16 20:07:22
0bd6d59f
WD
338+++ receiver.c 2004-07-17 21:27:55
339@@ -28,6 +28,7 @@ extern int max_delete;
340 extern int csum_length;
341 extern struct stats stats;
342 extern int dry_run;
343+extern int read_batch;
344 extern int am_server;
345 extern int relative_paths;
346 extern int keep_dirlinks;
347@@ -299,13 +300,38 @@ static int receive_data(int f_in,struct
348 return 1;
349 }
350
351+static char *read_gen_name(int fd, char *buf, char *realname)
352+{
353+ int len = read_byte(fd);
354+ if (len & 0x80) {
355+#if MAXPATHLEN > 32767
356+ uchar lenbuf[2];
357+ read_buf(fd, (char *)lenbuf, 2);
358+ len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
359+#else
360+ len = (len & ~0x80) * 0x100 + read_byte(fd);
361+#endif
362+ }
363+ if (len) {
364+ if (len >= MAXPATHLEN) {
365+ rprintf(FERROR, "bogus data on generator name pipe\n");
366+ exit_cleanup(RERR_PROTOCOL);
367+ }
368+ read_sbuf(fd, buf, len);
369+ return buf;
370+ }
371+ return realname;
372+}
373+
374
375 /**
77945073
WD
376 * main routine for receiver process.
377 *
378 * Receiver process runs on the same host as the generator process. */
9be39c35 379-int recv_files(int f_in, struct file_list *flist, char *local_name)
77945073 380+int recv_files(int f_in, struct file_list *flist, char *local_name,
0bd6d59f 381+ int f_name_in)
77945073 382 {
0bd6d59f 383+ int next_gen_i = -1;
77945073
WD
384 int fd1,fd2;
385 STRUCT_STAT st;
0bd6d59f
WD
386 char *fname, fbuf[MAXPATHLEN];
387@@ -332,8 +358,20 @@ int recv_files(int f_in, struct file_lis
0b32df42 388
0bd6d59f
WD
389 i = read_int(f_in);
390 if (i == -1) {
391+ if (read_batch) {
392+ if (next_gen_i < 0)
393+ next_gen_i = read_int(f_name_in);
394+ while (next_gen_i < flist->count) {
395+ read_gen_name(f_name_in, fnamecmpbuf,
396+ NULL);
397+ next_gen_i = read_int(f_name_in);
398+ }
399+ next_gen_i = -1;
400+ }
401+
402 if (phase)
403 break;
404+
405 phase = 1;
406 csum_length = SUM_LENGTH;
407 if (verbose > 2)
408@@ -373,19 +411,31 @@ int recv_files(int f_in, struct file_lis
77945073
WD
409 if (verbose > 2)
410 rprintf(FINFO,"recv_files(%s)\n",fname);
411
412- fnamecmp = fname;
0bd6d59f
WD
413+ if (read_batch) {
414+ if (next_gen_i < 0)
415+ next_gen_i = read_int(f_name_in);
416+ while (i > next_gen_i) {
417+ read_gen_name(f_name_in, fnamecmpbuf, NULL);
418+ next_gen_i = read_int(f_name_in);
419+ }
420+ if (i < next_gen_i) {
421+ rprintf(FINFO, "skipping update for %s\n",
422+ fname);
423+ receive_data(f_in,NULL,-1,NULL,file->length);
424+ continue;
425+ }
426+ next_gen_i = -1;
77945073 427+ }
0bd6d59f
WD
428+
429+ if (f_name_in >= 0)
430+ fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
431+ else
0b32df42 432+ fnamecmp = fname;
0bd6d59f 433+
77945073
WD
434
435 /* open the file */
436 fd1 = do_open(fnamecmp, O_RDONLY, 0);
437
438- if (fd1 == -1 && compare_dest != NULL) {
439- /* try the file at compare_dest instead */
440- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
441- compare_dest, fname);
442- fnamecmp = fnamecmpbuf;
443- fd1 = do_open(fnamecmp, O_RDONLY, 0);
444- }
445-
446 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
447 rsyserr(FERROR, errno, "fstat %s failed",
448 full_fname(fnamecmp));