[rsync/rsync-patches.git] / g2r-basis-filename.diff

Added a pipe from the generator to the receiver that communicates
what basis file we used to generate the file data (if it was not
the default name).  This optimizes away the basis-file search in
the receiver and makes future options that do more basis-file
searching more efficient (such as the --fuzzy option and the
support for multiple --compare-dest options).

Also fixes a potential synchronization problem between the generator
and the receiver in read-batch mode.  Should consider making the
sending of the index value the default for this pipe (it's currently
only sent in batch mode due to the sender not listening to the
generator to determine what files get updated).

You must run "make proto" before compiling.

--- orig/generator.c	2004-07-17 16:30:20
+++ generator.c	2004-07-17 15:50:09
@@ -251,11 +251,11 @@ static void generate_and_send_sums(struc
  * out.  It might be wrong.
  */
 static void recv_generator(char *fname, struct file_struct *file, int i,
-			   int f_out)
+			   int f_out, int f_nameout)
 {
-	int fd;
+	int fd = -1;
 	STRUCT_STAT st;
-	struct map_struct *mapbuf;
+	struct map_struct *mapbuf = NULL;
 	int statret;
 	char *fnamecmp;
 	char fnamecmpbuf[MAXPATHLEN];
@@ -398,9 +398,6 @@ static void recv_generator(char *fname, 
 	}
 #endif
 
-	if (read_batch)
-		return;
-
 	if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
 		return;
 
@@ -418,8 +415,10 @@ static void recv_generator(char *fname, 
 		statret = link_stat(fnamecmpbuf, &st, 0);
 		if (!S_ISREG(st.st_mode))
 			statret = -1;
-		if (statret == -1)
+		if (statret == -1) {
 			errno = saveerrno;
+			*fnamecmpbuf = '\0';
+		}
 #if HAVE_LINK
 		else if (link_dest && !dry_run) {
 			if (do_link(fnamecmpbuf, fname) != 0) {
@@ -427,22 +426,22 @@ static void recv_generator(char *fname, 
 					rsyserr(FINFO, errno, "link %s => %s",
 						fnamecmpbuf, fname);
 				}
-			}
-			fnamecmp = fnamecmpbuf;
+				fnamecmp = fnamecmpbuf;
+			} else
+				*fnamecmpbuf = '\0';
 		}
 #endif
 		else
 			fnamecmp = fnamecmpbuf;
-	}
+	} else
+		*fnamecmpbuf = '\0';
 
 	if (statret == -1) {
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
-		if (errno == ENOENT) {
-			write_int(f_out,i);
-			if (!dry_run)
-				write_sum_head(f_out, NULL);
-		} else if (verbose > 1) {
+		if (errno == ENOENT)
+			goto notify_sender;
+		if (verbose > 1) {
 			rsyserr(FERROR, errno,
 				"recv_generator: failed to open %s",
 				full_fname(fname));
@@ -451,26 +450,23 @@ static void recv_generator(char *fname, 
 	}
 
 	if (!S_ISREG(st.st_mode)) {
-		if (delete_file(fname) != 0) {
+		if (delete_file(fname) != 0)
 			return;
-		}
 
 		/* now pretend the file didn't exist */
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
-		write_int(f_out,i);
-		if (!dry_run)
-			write_sum_head(f_out, NULL);
-		return;
+		statret = -1;
+		goto notify_sender;
 	}
 
-	if (opt_ignore_existing && fnamecmp == fname) {
+	if (opt_ignore_existing && !*fnamecmpbuf) {
 		if (verbose > 1)
 			rprintf(FINFO,"%s exists\n",fname);
 		return;
 	}
 
-	if (update_only && fnamecmp == fname
+	if (update_only && !*fnamecmpbuf
 	    && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 		if (verbose > 1)
 			rprintf(FINFO,"%s is newer\n",fname);
@@ -478,21 +474,17 @@ static void recv_generator(char *fname, 
 	}
 
 	if (skip_file(fname, file, &st)) {
-		if (fnamecmp == fname)
+		if (!*fnamecmpbuf)
 			set_perms(fname, file, &st, PERMS_REPORT);
 		return;
 	}
 
-	if (dry_run) {
-		write_int(f_out,i);
-		return;
-	}
-
-	if (whole_file > 0) {
-		write_int(f_out,i);
-		write_sum_head(f_out, NULL);
-		return;
+	if (dry_run || whole_file) {
+		statret = -1;
+		goto notify_sender;
 	}
+	if (read_batch)
+		goto notify_sender;
 
 	/* open the file */
 	fd = do_open(fnamecmp, O_RDONLY, 0);
@@ -503,15 +495,12 @@ static void recv_generator(char *fname, 
 		/* pretend the file didn't exist */
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
-		write_int(f_out,i);
-		write_sum_head(f_out, NULL);
-		return;
+		statret = -1;
+		goto notify_sender;
 	}
 
 	if (st.st_size > 0)
 		mapbuf = map_file(fd,st.st_size);
-	else
-		mapbuf = NULL;
 
 	if (verbose > 3) {
 		rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
@@ -521,16 +510,43 @@ static void recv_generator(char *fname, 
 	if (verbose > 2)
 		rprintf(FINFO, "generating and sending sums for %d\n", i);
 
-	write_int(f_out,i);
-	generate_and_send_sums(mapbuf, st.st_size, f_out);
+notify_sender:
+	if (f_nameout >= 0) {
+		uchar lenbuf[3], *lb = lenbuf;
+		int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
+		if (read_batch)
+			write_int(f_nameout, i);
+		if (len > 0x7F) {
+#if MAXPATHLEN > 0x7FFF
+			*lb++ = len / 0x10000 + 0x80;
+			*lb++ = len / 0x100;
+#else
+			*lb++ = len / 0x100 + 0x80;
+#endif
+		}
+		*lb = len;
+		write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
+		if (len)
+			write_buf(f_nameout, fnamecmpbuf, len);
+	}
 
-	close(fd);
-	if (mapbuf)
-		unmap_file(mapbuf);
+	if (read_batch)
+		return;
+
+	write_int(f_out, i);
+	if (statret == 0) {
+		generate_and_send_sums(mapbuf, st.st_size, f_out);
+
+		close(fd);
+		if (mapbuf)
+			unmap_file(mapbuf);
+	} else if (!dry_run)
+		write_sum_head(f_out, NULL);
 }
 
 
-void generate_files(int f_out, struct file_list *flist, char *local_name)
+void generate_files(int f_out, struct file_list *flist, char *local_name,
+		    int f_nameout)
 {
 	int i;
 	int phase = 0;
@@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi
 		}
 
 		recv_generator(local_name ? local_name : f_name_to(file, fbuf),
-			       file, i, f_out);
+			       file, i, f_out, f_nameout);
 	}
 
 	phase++;
@@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi
 		rprintf(FINFO,"generate_files phase=%d\n",phase);
 
 	write_int(f_out, -1);
+	if (read_batch)
+		write_int(f_nameout, flist->count);
 
 	/* files can cycle through the system more than once
 	 * to catch initial checksum errors */
 	while ((i = get_redo_num()) != -1) {
 		struct file_struct *file = flist->files[i];
 		recv_generator(local_name ? local_name : f_name_to(file, fbuf),
-			       file, i, f_out);
+			       file, i, f_out, f_nameout);
 	}
 
 	phase++;
@@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi
 		rprintf(FINFO,"generate_files phase=%d\n",phase);
 
 	write_int(f_out, -1);
+	if (read_batch)
+		write_int(f_nameout, flist->count);
 
 	if (preserve_hard_links)
 		do_hard_links();
@@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi
 		if (!file->basename || !S_ISDIR(file->mode))
 			continue;
 		recv_generator(local_name ? local_name : f_name(file),
-			       file, i, -1);
+			       file, i, -1, -1);
 	}
 
 	if (verbose > 2)
--- orig/main.c	2004-07-17 15:20:05
+++ main.c	2004-07-17 15:58:11
@@ -57,6 +57,7 @@ extern int filesfrom_fd;
 extern pid_t cleanup_child_pid;
 extern char *files_from;
 extern char *remote_filesfrom_file;
+extern char *compare_dest;
 extern char *rsync_path;
 extern char *shell_cmd;
 extern char *batch_name;
@@ -444,7 +445,8 @@ static int do_recv(int f_in,int f_out,st
 {
 	int pid;
 	int status = 0;
-	int error_pipe[2];
+	int error_pipe[2], name_pipe[2];
+	int need_name_pipe = compare_dest || read_batch;
 
 	if (preserve_hard_links)
 		init_hard_links(flist);
@@ -456,8 +458,9 @@ static int do_recv(int f_in,int f_out,st
 		}
 	}
 
-	if (fd_pair(error_pipe) < 0) {
-		rprintf(FERROR,"error pipe failed in do_recv\n");
+	if (fd_pair(error_pipe) < 0
+	    || (need_name_pipe && fd_pair(name_pipe) < 0)) {
+		rprintf(FERROR, "fd_pair() failed in do_recv\n");
 		exit_cleanup(RERR_SOCKETIO);
 	}
 
@@ -465,6 +468,11 @@ static int do_recv(int f_in,int f_out,st
 
 	if ((pid = do_fork()) == 0) {
 		close(error_pipe[0]);
+		if (need_name_pipe) {
+			close(name_pipe[1]);
+			set_blocking(name_pipe[0]);
+		} else
+			name_pipe[0] = -1;
 		if (f_in != f_out)
 			close(f_out);
 
@@ -474,7 +482,7 @@ static int do_recv(int f_in,int f_out,st
 		/* set place to send errors */
 		set_msg_fd_out(error_pipe[1]);
 
-		recv_files(f_in,flist,local_name);
+		recv_files(f_in, flist, local_name, name_pipe[0]);
 		io_flush(FULL_FLUSH);
 		report(f_in);
 
@@ -492,6 +500,11 @@ static int do_recv(int f_in,int f_out,st
 		stop_write_batch();
 
 	close(error_pipe[1]);
+	if (need_name_pipe) {
+		close(name_pipe[0]);
+		set_nonblocking(name_pipe[1]);
+	} else
+		name_pipe[1] = -1;
 	if (f_in != f_out)
 		close(f_in);
 
@@ -499,7 +512,7 @@ static int do_recv(int f_in,int f_out,st
 
 	set_msg_fd_in(error_pipe[0]);
 
-	generate_files(f_out, flist, local_name);
+	generate_files(f_out, flist, local_name, name_pipe[1]);
 
 	get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 	report(-1);
--- orig/receiver.c	2004-07-16 20:07:22
+++ receiver.c	2004-07-17 21:27:55
@@ -28,6 +28,7 @@ extern int max_delete;
 extern int csum_length;
 extern struct stats stats;
 extern int dry_run;
+extern int read_batch;
 extern int am_server;
 extern int relative_paths;
 extern int keep_dirlinks;
@@ -299,13 +300,38 @@ static int receive_data(int f_in,struct 
 	return 1;
 }
 
+static char *read_gen_name(int fd, char *buf, char *realname)
+{
+	int len = read_byte(fd);
+	if (len & 0x80) {
+#if MAXPATHLEN > 32767
+		uchar lenbuf[2];
+		read_buf(fd, (char *)lenbuf, 2);
+		len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
+#else
+		len = (len & ~0x80) * 0x100 + read_byte(fd);
+#endif
+	}
+	if (len) {
+		if (len >= MAXPATHLEN) {
+			rprintf(FERROR, "bogus data on generator name pipe\n");
+			exit_cleanup(RERR_PROTOCOL);
+		}
+		read_sbuf(fd, buf, len);
+		return buf;
+	}
+	return realname;
+}
+
 
 /**
  * main routine for receiver process.
  *
  * Receiver process runs on the same host as the generator process. */
-int recv_files(int f_in, struct file_list *flist, char *local_name)
+int recv_files(int f_in, struct file_list *flist, char *local_name,
+	       int f_name_in)
 {
+	int next_gen_i = -1;
 	int fd1,fd2;
 	STRUCT_STAT st;
 	char *fname, fbuf[MAXPATHLEN];
@@ -332,8 +358,20 @@ int recv_files(int f_in, struct file_lis
 
 		i = read_int(f_in);
 		if (i == -1) {
+			if (read_batch) {
+				if (next_gen_i < 0)
+					next_gen_i = read_int(f_name_in);
+				while (next_gen_i < flist->count) {
+					read_gen_name(f_name_in, fnamecmpbuf,
+						      NULL);
+					next_gen_i = read_int(f_name_in);
+				}
+				next_gen_i = -1;
+			}
+
 			if (phase)
 				break;
+
 			phase = 1;
 			csum_length = SUM_LENGTH;
 			if (verbose > 2)
@@ -373,19 +411,31 @@ int recv_files(int f_in, struct file_lis
 		if (verbose > 2)
 			rprintf(FINFO,"recv_files(%s)\n",fname);
 
-		fnamecmp = fname;
+		if (read_batch) {
+			if (next_gen_i < 0)
+				next_gen_i = read_int(f_name_in);
+			while (i > next_gen_i) {
+				read_gen_name(f_name_in, fnamecmpbuf, NULL);
+				next_gen_i = read_int(f_name_in);
+			}
+			if (i < next_gen_i) {
+				rprintf(FINFO, "skipping update for %s\n",
+					fname);
+				receive_data(f_in,NULL,-1,NULL,file->length);
+				continue;
+			}
+			next_gen_i = -1;
+		}
+
+		if (f_name_in >= 0)
+			fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
+		else
+			fnamecmp = fname;
+
 
 		/* open the file */
 		fd1 = do_open(fnamecmp, O_RDONLY, 0);
 
-		if (fd1 == -1 && compare_dest != NULL) {
-			/* try the file at compare_dest instead */
-			pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
-				 compare_dest, fname);
-			fnamecmp = fnamecmpbuf;
-			fd1 = do_open(fnamecmp, O_RDONLY, 0);
-		}
-
 		if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 			rsyserr(FERROR, errno, "fstat %s failed",
 				full_fname(fnamecmp));
Commit	Line	Data
77945073 WD	1	Added a pipe from the generator to the receiver that communicates
	2	what basis file we used to generate the file data (if it was not
	3	the default name). This optimizes away the basis-file search in
	4	the receiver and makes future options that do more basis-file
	5	searching more efficient (such as the --fuzzy option and the
	6	support for multiple --compare-dest options).
	7
0bd6d59f WD	8	Also fixes a potential synchronization problem between the generator
	9	and the receiver in read-batch mode. Should consider making the
	10	sending of the index value the default for this pipe (it's currently
	11	only sent in batch mode due to the sender not listening to the
	12	generator to determine what files get updated).
	13
77945073 WD	14	You must run "make proto" before compiling.
77945073 WD	15
0bd6d59f WD	16	--- orig/generator.c 2004-07-17 16:30:20
	17	+++ generator.c 2004-07-17 15:50:09
	18	@@ -251,11 +251,11 @@ static void generate_and_send_sums(struc
77945073 WD	19	* out. It might be wrong.
	20	*/
	21	static void recv_generator(char fname, struct file_struct file, int i,
	22	- int f_out)
	23	+ int f_out, int f_nameout)
	24	{
0bd6d59f WD	25	- int fd;
0bd6d59f WD	26	+ int fd = -1;
77945073	27	STRUCT_STAT st;
0bd6d59f WD	28	- struct map_struct *mapbuf;
	29	+ struct map_struct *mapbuf = NULL;
	30	int statret;
	31	char *fnamecmp;
	32	char fnamecmpbuf[MAXPATHLEN];
	33	@@ -398,9 +398,6 @@ static void recv_generator(char *fname,
	34	}
	35	#endif
	36
	37	- if (read_batch)
	38	- return;
	39	-
	40	if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
	41	return;
	42
	43	@@ -418,8 +415,10 @@ static void recv_generator(char *fname,
77945073 WD	44	statret = link_stat(fnamecmpbuf, &st, 0);
	45	if (!S_ISREG(st.st_mode))
	46	statret = -1;
	47	- if (statret == -1)
0bd6d59f	48	+ if (statret == -1) {
77945073 WD	49	errno = saveerrno;
	50	+ *fnamecmpbuf = '\0';
	51	+ }
	52	#if HAVE_LINK
	53	else if (link_dest && !dry_run) {
	54	if (do_link(fnamecmpbuf, fname) != 0) {
0bd6d59f	55	@@ -427,22 +426,22 @@ static void recv_generator(char *fname,
77945073 WD	56	rsyserr(FINFO, errno, "link %s => %s",
	57	fnamecmpbuf, fname);
	58	}
	59	- }
	60	- fnamecmp = fnamecmpbuf;
	61	+ fnamecmp = fnamecmpbuf;
	62	+ } else
	63	+ *fnamecmpbuf = '\0';
	64	}
	65	#endif
	66	else
	67	fnamecmp = fnamecmpbuf;
	68	- }
	69	+ } else
	70	+ *fnamecmpbuf = '\0';
	71
	72	if (statret == -1) {
	73	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	74	return;
0bd6d59f WD	75	- if (errno == ENOENT) {
	76	- write_int(f_out,i);
	77	- if (!dry_run)
	78	- write_sum_head(f_out, NULL);
	79	- } else if (verbose > 1) {
	80	+ if (errno == ENOENT)
	81	+ goto notify_sender;
	82	+ if (verbose > 1) {
	83	rsyserr(FERROR, errno,
	84	"recv_generator: failed to open %s",
	85	full_fname(fname));
	86	@@ -451,26 +450,23 @@ static void recv_generator(char *fname,
	87	}
	88
	89	if (!S_ISREG(st.st_mode)) {
	90	- if (delete_file(fname) != 0) {
	91	+ if (delete_file(fname) != 0)
	92	return;
	93	- }
	94
77945073 WD	95	/* now pretend the file didn't exist */
	96	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	97	return;
0bd6d59f WD	98	- write_int(f_out,i);
	99	- if (!dry_run)
	100	- write_sum_head(f_out, NULL);
	101	- return;
	102	+ statret = -1;
	103	+ goto notify_sender;
77945073 WD	104	}
	105
	106	- if (opt_ignore_existing && fnamecmp == fname) {
	107	+ if (opt_ignore_existing && !*fnamecmpbuf) {
	108	if (verbose > 1)
	109	rprintf(FINFO,"%s exists\n",fname);
	110	return;
	111	}
	112
	113	- if (update_only && fnamecmp == fname
	114	+ if (update_only && !*fnamecmpbuf
	115	&& cmp_modtime(st.st_mtime, file->modtime) > 0) {
	116	if (verbose > 1)
	117	rprintf(FINFO,"%s is newer\n",fname);
0bd6d59f	118	@@ -478,21 +474,17 @@ static void recv_generator(char *fname,
77945073 WD	119	}
	120
	121	if (skip_file(fname, file, &st)) {
	122	- if (fnamecmp == fname)
	123	+ if (!*fnamecmpbuf)
	124	set_perms(fname, file, &st, PERMS_REPORT);
	125	return;
	126	}
	127
0bd6d59f WD	128	- if (dry_run) {
	129	- write_int(f_out,i);
	130	- return;
	131	- }
	132	-
	133	- if (whole_file > 0) {
	134	- write_int(f_out,i);
	135	- write_sum_head(f_out, NULL);
	136	- return;
	137	+ if (dry_run \|\| whole_file) {
	138	+ statret = -1;
	139	+ goto notify_sender;
77945073	140	}
0bd6d59f WD	141	+ if (read_batch)
0bd6d59f WD	142	+ goto notify_sender;
77945073	143
0bd6d59f WD	144	/* open the file */
	145	fd = do_open(fnamecmp, O_RDONLY, 0);
	146	@@ -503,15 +495,12 @@ static void recv_generator(char *fname,
77945073 WD	147	/* pretend the file didn't exist */
	148	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	149	return;
0bd6d59f WD	150	- write_int(f_out,i);
	151	- write_sum_head(f_out, NULL);
	152	- return;
	153	+ statret = -1;
	154	+ goto notify_sender;
	155	}
	156
	157	if (st.st_size > 0)
	158	mapbuf = map_file(fd,st.st_size);
	159	- else
	160	- mapbuf = NULL;
	161
	162	if (verbose > 3) {
	163	rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
	164	@@ -521,16 +510,43 @@ static void recv_generator(char *fname,
77945073 WD	165	if (verbose > 2)
	166	rprintf(FINFO, "generating and sending sums for %d\n", i);
	167
0bd6d59f WD	168	- write_int(f_out,i);
	169	- generate_and_send_sums(mapbuf, st.st_size, f_out);
	170	+notify_sender:
0b32df42 WD	171	+ if (f_nameout >= 0) {
0b32df42 WD	172	+ uchar lenbuf[3], *lb = lenbuf;
0bd6d59f WD	173	+ int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
	174	+ if (read_batch)
	175	+ write_int(f_nameout, i);
	176	+ if (len > 0x7F) {
	177	+#if MAXPATHLEN > 0x7FFF
	178	+ *lb++ = len / 0x10000 + 0x80;
	179	+ *lb++ = len / 0x100;
0b32df42	180	+#else
0bd6d59f	181	+ *lb++ = len / 0x100 + 0x80;
0b32df42 WD	182	+#endif
	183	+ }
	184	+ *lb = len;
	185	+ write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
0bd6d59f WD	186	+ if (len)
0bd6d59f WD	187	+ write_buf(f_nameout, fnamecmpbuf, len);
0b32df42	188	+ }
77945073	189
0bd6d59f WD	190	- close(fd);
	191	- if (mapbuf)
	192	- unmap_file(mapbuf);
	193	+ if (read_batch)
	194	+ return;
	195	+
	196	+ write_int(f_out, i);
	197	+ if (statret == 0) {
	198	+ generate_and_send_sums(mapbuf, st.st_size, f_out);
	199	+
	200	+ close(fd);
	201	+ if (mapbuf)
	202	+ unmap_file(mapbuf);
	203	+ } else if (!dry_run)
	204	+ write_sum_head(f_out, NULL);
77945073 WD	205	}
	206
	207
9be39c35 WD	208	-void generate_files(int f_out, struct file_list flist, char local_name)
9be39c35 WD	209	+void generate_files(int f_out, struct file_list flist, char local_name,
77945073 WD	210	+ int f_nameout)
	211	{
	212	int i;
8cec1ead	213	int phase = 0;
0bd6d59f	214	@@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi
77945073 WD	215	}
	216
	217	recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35 WD	218	- file, i, f_out);
9be39c35 WD	219	+ file, i, f_out, f_nameout);
77945073 WD	220	}
	221
	222	phase++;
0bd6d59f WD	223	@@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi
	224	rprintf(FINFO,"generate_files phase=%d\n",phase);
	225
	226	write_int(f_out, -1);
	227	+ if (read_batch)
	228	+ write_int(f_nameout, flist->count);
	229
	230	/* files can cycle through the system more than once
	231	* to catch initial checksum errors */
77945073 WD	232	while ((i = get_redo_num()) != -1) {
	233	struct file_struct *file = flist->files[i];
	234	recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35 WD	235	- file, i, f_out);
9be39c35 WD	236	+ file, i, f_out, f_nameout);
77945073 WD	237	}
	238
	239	phase++;
0bd6d59f WD	240	@@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi
	241	rprintf(FINFO,"generate_files phase=%d\n",phase);
	242
	243	write_int(f_out, -1);
	244	+ if (read_batch)
	245	+ write_int(f_nameout, flist->count);
	246
	247	if (preserve_hard_links)
	248	do_hard_links();
	249	@@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi
77945073 WD	250	if (!file->basename \|\| !S_ISDIR(file->mode))
	251	continue;
	252	recv_generator(local_name ? local_name : f_name(file),
	253	- file, i, -1);
	254	+ file, i, -1, -1);
	255	}
	256
	257	if (verbose > 2)
0b32df42	258	--- orig/main.c 2004-07-17 15:20:05
0bd6d59f WD	259	+++ main.c 2004-07-17 15:58:11
	260	@@ -57,6 +57,7 @@ extern int filesfrom_fd;
	261	extern pid_t cleanup_child_pid;
	262	extern char *files_from;
	263	extern char *remote_filesfrom_file;
	264	+extern char *compare_dest;
	265	extern char *rsync_path;
	266	extern char *shell_cmd;
	267	extern char *batch_name;
	268	@@ -444,7 +445,8 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	269	{
77945073 WD	270	int pid;
8cec1ead	271	int status = 0;
77945073	272	- int error_pipe[2];
77945073	273	+ int error_pipe[2], name_pipe[2];
0bd6d59f	274	+ int need_name_pipe = compare_dest \|\| read_batch;
77945073 WD	275
	276	if (preserve_hard_links)
	277	init_hard_links(flist);
0bd6d59f	278	@@ -456,8 +458,9 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	279	}
	280	}
	281
	282	- if (fd_pair(error_pipe) < 0) {
	283	- rprintf(FERROR,"error pipe failed in do_recv\n");
0bd6d59f WD	284	+ if (fd_pair(error_pipe) < 0
0bd6d59f WD	285	+ \|\| (need_name_pipe && fd_pair(name_pipe) < 0)) {
77945073 WD	286	+ rprintf(FERROR, "fd_pair() failed in do_recv\n");
	287	exit_cleanup(RERR_SOCKETIO);
	288	}
	289
0bd6d59f	290	@@ -465,6 +468,11 @@ static int do_recv(int f_in,int f_out,st
77945073	291
8cec1ead	292	if ((pid = do_fork()) == 0) {
77945073	293	close(error_pipe[0]);
0bd6d59f WD	294	+ if (need_name_pipe) {
	295	+ close(name_pipe[1]);
	296	+ set_blocking(name_pipe[0]);
	297	+ } else
	298	+ name_pipe[0] = -1;
77945073 WD	299	if (f_in != f_out)
77945073 WD	300	close(f_out);
77945073	301
0bd6d59f	302	@@ -474,7 +482,7 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	303	/* set place to send errors */
	304	set_msg_fd_out(error_pipe[1]);
	305
	306	- recv_files(f_in,flist,local_name);
	307	+ recv_files(f_in, flist, local_name, name_pipe[0]);
	308	io_flush(FULL_FLUSH);
	309	report(f_in);
	310
0bd6d59f	311	@@ -492,6 +500,11 @@ static int do_recv(int f_in,int f_out,st
9be39c35	312	stop_write_batch();
77945073 WD	313
77945073 WD	314	close(error_pipe[1]);
0bd6d59f WD	315	+ if (need_name_pipe) {
	316	+ close(name_pipe[0]);
	317	+ set_nonblocking(name_pipe[1]);
	318	+ } else
	319	+ name_pipe[1] = -1;
77945073 WD	320	if (f_in != f_out)
77945073 WD	321	close(f_in);
77945073	322
0bd6d59f	323	@@ -499,7 +512,7 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	324
	325	set_msg_fd_in(error_pipe[0]);
	326
	327	- generate_files(f_out, flist, local_name);
	328	+ generate_files(f_out, flist, local_name, name_pipe[1]);
	329
	330	get_redo_num(); /* Read final MSG_DONE and any prior messages. */
	331	report(-1);
f6c3b300	332	--- orig/receiver.c 2004-07-16 20:07:22
0bd6d59f WD	333	+++ receiver.c 2004-07-17 21:27:55
	334	@@ -28,6 +28,7 @@ extern int max_delete;
	335	extern int csum_length;
	336	extern struct stats stats;
	337	extern int dry_run;
	338	+extern int read_batch;
	339	extern int am_server;
	340	extern int relative_paths;
	341	extern int keep_dirlinks;
	342	@@ -299,13 +300,38 @@ static int receive_data(int f_in,struct
	343	return 1;
	344	}
	345
	346	+static char read_gen_name(int fd, char buf, char *realname)
	347	+{
	348	+ int len = read_byte(fd);
	349	+ if (len & 0x80) {
	350	+#if MAXPATHLEN > 32767
	351	+ uchar lenbuf[2];
	352	+ read_buf(fd, (char *)lenbuf, 2);
	353	+ len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
	354	+#else
	355	+ len = (len & ~0x80) * 0x100 + read_byte(fd);
	356	+#endif
	357	+ }
	358	+ if (len) {
	359	+ if (len >= MAXPATHLEN) {
	360	+ rprintf(FERROR, "bogus data on generator name pipe\n");
	361	+ exit_cleanup(RERR_PROTOCOL);
	362	+ }
	363	+ read_sbuf(fd, buf, len);
	364	+ return buf;
	365	+ }
	366	+ return realname;
	367	+}
	368	+
	369
	370	/**
77945073 WD	371	* main routine for receiver process.
	372	*
	373	* Receiver process runs on the same host as the generator process. */
9be39c35	374	-int recv_files(int f_in, struct file_list flist, char local_name)
77945073	375	+int recv_files(int f_in, struct file_list flist, char local_name,
0bd6d59f	376	+ int f_name_in)
77945073	377	{
0bd6d59f	378	+ int next_gen_i = -1;
77945073 WD	379	int fd1,fd2;
77945073 WD	380	STRUCT_STAT st;
0bd6d59f WD	381	char *fname, fbuf[MAXPATHLEN];
0bd6d59f WD	382	@@ -332,8 +358,20 @@ int recv_files(int f_in, struct file_lis
0b32df42	383
0bd6d59f WD	384	i = read_int(f_in);
	385	if (i == -1) {
	386	+ if (read_batch) {
	387	+ if (next_gen_i < 0)
	388	+ next_gen_i = read_int(f_name_in);
	389	+ while (next_gen_i < flist->count) {
	390	+ read_gen_name(f_name_in, fnamecmpbuf,
	391	+ NULL);
	392	+ next_gen_i = read_int(f_name_in);
	393	+ }
	394	+ next_gen_i = -1;
	395	+ }
	396	+
	397	if (phase)
	398	break;
	399	+
	400	phase = 1;
	401	csum_length = SUM_LENGTH;
	402	if (verbose > 2)
	403	@@ -373,19 +411,31 @@ int recv_files(int f_in, struct file_lis
77945073 WD	404	if (verbose > 2)
	405	rprintf(FINFO,"recv_files(%s)\n",fname);
	406
	407	- fnamecmp = fname;
0bd6d59f WD	408	+ if (read_batch) {
	409	+ if (next_gen_i < 0)
	410	+ next_gen_i = read_int(f_name_in);
	411	+ while (i > next_gen_i) {
	412	+ read_gen_name(f_name_in, fnamecmpbuf, NULL);
	413	+ next_gen_i = read_int(f_name_in);
	414	+ }
	415	+ if (i < next_gen_i) {
	416	+ rprintf(FINFO, "skipping update for %s\n",
	417	+ fname);
	418	+ receive_data(f_in,NULL,-1,NULL,file->length);
	419	+ continue;
	420	+ }
	421	+ next_gen_i = -1;
77945073	422	+ }
0bd6d59f WD	423	+
	424	+ if (f_name_in >= 0)
	425	+ fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
	426	+ else
0b32df42	427	+ fnamecmp = fname;
0bd6d59f	428	+
77945073 WD	429
	430	/* open the file */
	431	fd1 = do_open(fnamecmp, O_RDONLY, 0);
	432
	433	- if (fd1 == -1 && compare_dest != NULL) {
	434	- /* try the file at compare_dest instead */
	435	- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
	436	- compare_dest, fname);
	437	- fnamecmp = fnamecmpbuf;
	438	- fd1 = do_open(fnamecmp, O_RDONLY, 0);
	439	- }
	440	-
	441	if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
	442	rsyserr(FERROR, errno, "fstat %s failed",
	443	full_fname(fnamecmp));