[rsync/rsync-patches.git] / g2r-basis-filename.diff

Added a pipe from the generator to the receiver that communicates
what basis file we used to generate the file data (if it was not
the default name).  This optimizes away the basis-file search in
the receiver and makes future options that do more basis-file
searching more efficient (such as the --fuzzy option and the
support for multiple --compare-dest options).

You must run "make proto" before compiling.

--- orig/generator.c	2004-07-17 15:20:05
+++ generator.c	2004-07-17 10:23:13
@@ -251,7 +251,7 @@ static void generate_and_send_sums(struc
  * out.  It might be wrong.
  */
 static void recv_generator(char *fname, struct file_struct *file, int i,
-			   int f_out)
+			   int f_out, int f_nameout)
 {
 	int fd;
 	STRUCT_STAT st;
@@ -418,8 +418,10 @@ static void recv_generator(char *fname, 
 		statret = link_stat(fnamecmpbuf, &st, 0);
 		if (!S_ISREG(st.st_mode))
 			statret = -1;
-		if (statret == -1)
+		if (statret < 0) {
 			errno = saveerrno;
+			*fnamecmpbuf = '\0';
+		}
 #if HAVE_LINK
 		else if (link_dest && !dry_run) {
 			if (do_link(fnamecmpbuf, fname) != 0) {
@@ -427,18 +429,22 @@ static void recv_generator(char *fname, 
 					rsyserr(FINFO, errno, "link %s => %s",
 						fnamecmpbuf, fname);
 				}
-			}
-			fnamecmp = fnamecmpbuf;
+				fnamecmp = fnamecmpbuf;
+			} else
+				*fnamecmpbuf = '\0';
 		}
 #endif
 		else
 			fnamecmp = fnamecmpbuf;
-	}
+	} else
+		*fnamecmpbuf = '\0';
 
 	if (statret == -1) {
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
 		if (errno == ENOENT) {
+			if (f_nameout >= 0)
+				write_byte(f_nameout, 0);
 			write_int(f_out,i);
 			if (!dry_run)
 				write_sum_head(f_out, NULL);
@@ -458,19 +464,21 @@ static void recv_generator(char *fname, 
 		/* now pretend the file didn't exist */
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
+		if (f_nameout >= 0)
+			write_byte(f_nameout, 0);
 		write_int(f_out,i);
 		if (!dry_run)
 			write_sum_head(f_out, NULL);
 		return;
 	}
 
-	if (opt_ignore_existing && fnamecmp == fname) {
+	if (opt_ignore_existing && !*fnamecmpbuf) {
 		if (verbose > 1)
 			rprintf(FINFO,"%s exists\n",fname);
 		return;
 	}
 
-	if (update_only && fnamecmp == fname
+	if (update_only && !*fnamecmpbuf
 	    && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 		if (verbose > 1)
 			rprintf(FINFO,"%s is newer\n",fname);
@@ -478,17 +486,21 @@ static void recv_generator(char *fname, 
 	}
 
 	if (skip_file(fname, file, &st)) {
-		if (fnamecmp == fname)
+		if (!*fnamecmpbuf)
 			set_perms(fname, file, &st, PERMS_REPORT);
 		return;
 	}
 
 	if (dry_run) {
+		if (f_nameout >= 0)
+			write_byte(f_nameout, 0);
 		write_int(f_out,i);
 		return;
 	}
 
 	if (whole_file > 0) {
+		if (f_nameout >= 0)
+			write_byte(f_nameout, 0);
 		write_int(f_out,i);
 		write_sum_head(f_out, NULL);
 		return;
@@ -503,6 +515,8 @@ static void recv_generator(char *fname, 
 		/* pretend the file didn't exist */
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
+		if (f_nameout >= 0)
+			write_byte(f_nameout, 0);
 		write_int(f_out,i);
 		write_sum_head(f_out, NULL);
 		return;
@@ -521,6 +535,22 @@ static void recv_generator(char *fname, 
 	if (verbose > 2)
 		rprintf(FINFO, "generating and sending sums for %d\n", i);
 
+	if (f_nameout >= 0) {
+		uchar lenbuf[3], *lb = lenbuf;
+		int len = strlen(fnamecmpbuf);
+		if (len > 127) {
+#if MAXPATHLEN > 32767
+		    *lb++ = len / 0x10000 + 0x80;
+		    *lb++ = len / 0x100;
+#else
+		    *lb++ = len / 0x100 + 0x80;
+#endif
+		}
+		*lb = len;
+		write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
+		write_buf(f_nameout, fnamecmpbuf, len);
+	}
+
 	write_int(f_out,i);
 	generate_and_send_sums(mapbuf, st.st_size, f_out);
 
@@ -530,7 +560,8 @@ static void recv_generator(char *fname, 
 }
 
 
-void generate_files(int f_out, struct file_list *flist, char *local_name)
+void generate_files(int f_out, struct file_list *flist, char *local_name,
+		    int f_nameout)
 {
 	int i;
 	int phase = 0;
@@ -571,7 +602,7 @@ void generate_files(int f_out, struct fi
 		}
 
 		recv_generator(local_name ? local_name : f_name_to(file, fbuf),
-			       file, i, f_out);
+			       file, i, f_out, f_nameout);
 	}
 
 	phase++;
@@ -588,7 +619,7 @@ void generate_files(int f_out, struct fi
 	while ((i = get_redo_num()) != -1) {
 		struct file_struct *file = flist->files[i];
 		recv_generator(local_name ? local_name : f_name_to(file, fbuf),
-			       file, i, f_out);
+			       file, i, f_out, f_nameout);
 	}
 
 	phase++;
@@ -607,7 +638,7 @@ void generate_files(int f_out, struct fi
 		if (!file->basename || !S_ISDIR(file->mode))
 			continue;
 		recv_generator(local_name ? local_name : f_name(file),
-			       file, i, -1);
+			       file, i, -1, -1);
 	}
 
 	if (verbose > 2)
--- orig/main.c	2004-07-17 15:20:05
+++ main.c	2004-07-17 15:22:08
@@ -444,7 +444,7 @@ static int do_recv(int f_in,int f_out,st
 {
 	int pid;
 	int status = 0;
-	int error_pipe[2];
+	int error_pipe[2], name_pipe[2];
 
 	if (preserve_hard_links)
 		init_hard_links(flist);
@@ -456,8 +456,8 @@ static int do_recv(int f_in,int f_out,st
 		}
 	}
 
-	if (fd_pair(error_pipe) < 0) {
-		rprintf(FERROR,"error pipe failed in do_recv\n");
+	if (fd_pair(error_pipe) < 0 || fd_pair(name_pipe) < 0) {
+		rprintf(FERROR, "fd_pair() failed in do_recv\n");
 		exit_cleanup(RERR_SOCKETIO);
 	}
 
@@ -465,8 +465,10 @@ static int do_recv(int f_in,int f_out,st
 
 	if ((pid = do_fork()) == 0) {
 		close(error_pipe[0]);
+		close(name_pipe[1]);
 		if (f_in != f_out)
 			close(f_out);
+		set_blocking(name_pipe[0]);
 
 		/* we can't let two processes write to the socket at one time */
 		io_multiplexing_close();
@@ -474,7 +476,7 @@ static int do_recv(int f_in,int f_out,st
 		/* set place to send errors */
 		set_msg_fd_out(error_pipe[1]);
 
-		recv_files(f_in,flist,local_name);
+		recv_files(f_in, flist, local_name, name_pipe[0]);
 		io_flush(FULL_FLUSH);
 		report(f_in);
 
@@ -492,14 +494,16 @@ static int do_recv(int f_in,int f_out,st
 		stop_write_batch();
 
 	close(error_pipe[1]);
+	close(name_pipe[0]);
 	if (f_in != f_out)
 		close(f_in);
+	set_nonblocking(name_pipe[1]);
 
 	io_start_buffering_out();
 
 	set_msg_fd_in(error_pipe[0]);
 
-	generate_files(f_out, flist, local_name);
+	generate_files(f_out, flist, local_name, name_pipe[1]);
 
 	get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 	report(-1);
--- orig/receiver.c	2004-07-16 20:07:22
+++ receiver.c	2004-07-17 11:05:36
@@ -304,7 +304,8 @@ static int receive_data(int f_in,struct 
  * main routine for receiver process.
  *
  * Receiver process runs on the same host as the generator process. */
-int recv_files(int f_in, struct file_list *flist, char *local_name)
+int recv_files(int f_in, struct file_list *flist, char *local_name,
+	       int f_name)
 {
 	int fd1,fd2;
 	STRUCT_STAT st;
@@ -317,7 +318,7 @@ int recv_files(int f_in, struct file_lis
 	struct file_struct *file;
 	struct stats initial_stats;
 	int save_make_backups = make_backups;
-	int i, recv_ok, phase = 0;
+	int i, len, recv_ok, phase = 0;
 
 	if (verbose > 2)
 		rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
@@ -373,19 +374,25 @@ int recv_files(int f_in, struct file_lis
 		if (verbose > 2)
 			rprintf(FINFO,"recv_files(%s)\n",fname);
 
-		fnamecmp = fname;
+		len = read_byte(f_name);
+		if (len & 0x80) {
+#if MAXPATHLEN > 32767
+		    read_buf(f_name, fnamecmpbuf, 2);
+		    len = (len & ~0x80) * 0x10000
+			+ fnamecmpbuf[0] * 0x100 + fnamecmpbuf[1];
+#else
+		    len = (len & ~0x80) * 0x100 + read_byte(f_name);
+#endif
+		}
+		if (len) {
+			read_sbuf(f_name, fnamecmpbuf, len);
+			fnamecmp = fnamecmpbuf;
+		} else
+			fnamecmp = fname;
 
 		/* open the file */
 		fd1 = do_open(fnamecmp, O_RDONLY, 0);
 
-		if (fd1 == -1 && compare_dest != NULL) {
-			/* try the file at compare_dest instead */
-			pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
-				 compare_dest, fname);
-			fnamecmp = fnamecmpbuf;
-			fd1 = do_open(fnamecmp, O_RDONLY, 0);
-		}
-
 		if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 			rsyserr(FERROR, errno, "fstat %s failed",
 				full_fname(fnamecmp));
Commit	Line	Data
77945073 WD	1	Added a pipe from the generator to the receiver that communicates
	2	what basis file we used to generate the file data (if it was not
	3	the default name). This optimizes away the basis-file search in
	4	the receiver and makes future options that do more basis-file
	5	searching more efficient (such as the --fuzzy option and the
	6	support for multiple --compare-dest options).
	7
	8	You must run "make proto" before compiling.
	9
0b32df42 WD	10	--- orig/generator.c 2004-07-17 15:20:05
0b32df42 WD	11	+++ generator.c 2004-07-17 10:23:13
f6c3b300	12	@@ -251,7 +251,7 @@ static void generate_and_send_sums(struc
77945073 WD	13	* out. It might be wrong.
	14	*/
	15	static void recv_generator(char fname, struct file_struct file, int i,
	16	- int f_out)
	17	+ int f_out, int f_nameout)
	18	{
	19	int fd;
	20	STRUCT_STAT st;
f6c3b300	21	@@ -418,8 +418,10 @@ static void recv_generator(char *fname,
77945073 WD	22	statret = link_stat(fnamecmpbuf, &st, 0);
	23	if (!S_ISREG(st.st_mode))
	24	statret = -1;
	25	- if (statret == -1)
	26	+ if (statret < 0) {
	27	errno = saveerrno;
	28	+ *fnamecmpbuf = '\0';
	29	+ }
	30	#if HAVE_LINK
	31	else if (link_dest && !dry_run) {
	32	if (do_link(fnamecmpbuf, fname) != 0) {
f6c3b300	33	@@ -427,18 +429,22 @@ static void recv_generator(char *fname,
77945073 WD	34	rsyserr(FINFO, errno, "link %s => %s",
	35	fnamecmpbuf, fname);
	36	}
	37	- }
	38	- fnamecmp = fnamecmpbuf;
	39	+ fnamecmp = fnamecmpbuf;
	40	+ } else
	41	+ *fnamecmpbuf = '\0';
	42	}
	43	#endif
	44	else
	45	fnamecmp = fnamecmpbuf;
	46	- }
	47	+ } else
	48	+ *fnamecmpbuf = '\0';
	49
	50	if (statret == -1) {
	51	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	52	return;
	53	if (errno == ENOENT) {
	54	+ if (f_nameout >= 0)
0b32df42	55	+ write_byte(f_nameout, 0);
77945073 WD	56	write_int(f_out,i);
	57	if (!dry_run)
	58	write_sum_head(f_out, NULL);
f6c3b300	59	@@ -458,19 +464,21 @@ static void recv_generator(char *fname,
77945073 WD	60	/* now pretend the file didn't exist */
	61	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	62	return;
	63	+ if (f_nameout >= 0)
0b32df42	64	+ write_byte(f_nameout, 0);
77945073 WD	65	write_int(f_out,i);
	66	if (!dry_run)
	67	write_sum_head(f_out, NULL);
	68	return;
	69	}
	70
	71	- if (opt_ignore_existing && fnamecmp == fname) {
	72	+ if (opt_ignore_existing && !*fnamecmpbuf) {
	73	if (verbose > 1)
	74	rprintf(FINFO,"%s exists\n",fname);
	75	return;
	76	}
	77
	78	- if (update_only && fnamecmp == fname
	79	+ if (update_only && !*fnamecmpbuf
	80	&& cmp_modtime(st.st_mtime, file->modtime) > 0) {
	81	if (verbose > 1)
	82	rprintf(FINFO,"%s is newer\n",fname);
f6c3b300	83	@@ -478,17 +486,21 @@ static void recv_generator(char *fname,
77945073 WD	84	}
	85
	86	if (skip_file(fname, file, &st)) {
	87	- if (fnamecmp == fname)
	88	+ if (!*fnamecmpbuf)
	89	set_perms(fname, file, &st, PERMS_REPORT);
	90	return;
	91	}
	92
	93	if (dry_run) {
	94	+ if (f_nameout >= 0)
0b32df42	95	+ write_byte(f_nameout, 0);
77945073 WD	96	write_int(f_out,i);
	97	return;
	98	}
	99
9be39c35	100	if (whole_file > 0) {
77945073	101	+ if (f_nameout >= 0)
0b32df42	102	+ write_byte(f_nameout, 0);
77945073 WD	103	write_int(f_out,i);
	104	write_sum_head(f_out, NULL);
	105	return;
f6c3b300	106	@@ -503,6 +515,8 @@ static void recv_generator(char *fname,
77945073 WD	107	/* pretend the file didn't exist */
	108	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
	109	return;
	110	+ if (f_nameout >= 0)
0b32df42	111	+ write_byte(f_nameout, 0);
77945073 WD	112	write_int(f_out,i);
	113	write_sum_head(f_out, NULL);
	114	return;
0b32df42	115	@@ -521,6 +535,22 @@ static void recv_generator(char *fname,
77945073 WD	116	if (verbose > 2)
	117	rprintf(FINFO, "generating and sending sums for %d\n", i);
	118
0b32df42 WD	119	+ if (f_nameout >= 0) {
	120	+ uchar lenbuf[3], *lb = lenbuf;
	121	+ int len = strlen(fnamecmpbuf);
	122	+ if (len > 127) {
	123	+#if MAXPATHLEN > 32767
	124	+ *lb++ = len / 0x10000 + 0x80;
	125	+ *lb++ = len / 0x100;
	126	+#else
	127	+ *lb++ = len / 0x100 + 0x80;
	128	+#endif
	129	+ }
	130	+ *lb = len;
	131	+ write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
	132	+ write_buf(f_nameout, fnamecmpbuf, len);
	133	+ }
	134	+
77945073 WD	135	write_int(f_out,i);
	136	generate_and_send_sums(mapbuf, st.st_size, f_out);
	137
0b32df42	138	@@ -530,7 +560,8 @@ static void recv_generator(char *fname,
77945073 WD	139	}
	140
	141
9be39c35 WD	142	-void generate_files(int f_out, struct file_list flist, char local_name)
9be39c35 WD	143	+void generate_files(int f_out, struct file_list flist, char local_name,
77945073 WD	144	+ int f_nameout)
	145	{
	146	int i;
8cec1ead	147	int phase = 0;
0b32df42	148	@@ -571,7 +602,7 @@ void generate_files(int f_out, struct fi
77945073 WD	149	}
	150
	151	recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35 WD	152	- file, i, f_out);
9be39c35 WD	153	+ file, i, f_out, f_nameout);
77945073 WD	154	}
	155
	156	phase++;
0b32df42	157	@@ -588,7 +619,7 @@ void generate_files(int f_out, struct fi
77945073 WD	158	while ((i = get_redo_num()) != -1) {
	159	struct file_struct *file = flist->files[i];
	160	recv_generator(local_name ? local_name : f_name_to(file, fbuf),
9be39c35 WD	161	- file, i, f_out);
9be39c35 WD	162	+ file, i, f_out, f_nameout);
77945073 WD	163	}
	164
	165	phase++;
0b32df42	166	@@ -607,7 +638,7 @@ void generate_files(int f_out, struct fi
77945073 WD	167	if (!file->basename \|\| !S_ISDIR(file->mode))
	168	continue;
	169	recv_generator(local_name ? local_name : f_name(file),
	170	- file, i, -1);
	171	+ file, i, -1, -1);
	172	}
	173
	174	if (verbose > 2)
0b32df42 WD	175	--- orig/main.c 2004-07-17 15:20:05
	176	+++ main.c 2004-07-17 15:22:08
	177	@@ -444,7 +444,7 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	178	{
77945073 WD	179	int pid;
8cec1ead	180	int status = 0;
77945073	181	- int error_pipe[2];
77945073 WD	182	+ int error_pipe[2], name_pipe[2];
	183
	184	if (preserve_hard_links)
	185	init_hard_links(flist);
0b32df42	186	@@ -456,8 +456,8 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	187	}
	188	}
	189
	190	- if (fd_pair(error_pipe) < 0) {
	191	- rprintf(FERROR,"error pipe failed in do_recv\n");
	192	+ if (fd_pair(error_pipe) < 0 \|\| fd_pair(name_pipe) < 0) {
	193	+ rprintf(FERROR, "fd_pair() failed in do_recv\n");
	194	exit_cleanup(RERR_SOCKETIO);
	195	}
	196
0b32df42	197	@@ -465,8 +465,10 @@ static int do_recv(int f_in,int f_out,st
77945073	198
8cec1ead	199	if ((pid = do_fork()) == 0) {
77945073 WD	200	close(error_pipe[0]);
	201	+ close(name_pipe[1]);
	202	if (f_in != f_out)
	203	close(f_out);
	204	+ set_blocking(name_pipe[0]);
	205
	206	/* we can't let two processes write to the socket at one time */
	207	io_multiplexing_close();
0b32df42	208	@@ -474,7 +476,7 @@ static int do_recv(int f_in,int f_out,st
77945073 WD	209	/* set place to send errors */
	210	set_msg_fd_out(error_pipe[1]);
	211
	212	- recv_files(f_in,flist,local_name);
	213	+ recv_files(f_in, flist, local_name, name_pipe[0]);
	214	io_flush(FULL_FLUSH);
	215	report(f_in);
	216
0b32df42	217	@@ -492,14 +494,16 @@ static int do_recv(int f_in,int f_out,st
9be39c35	218	stop_write_batch();
77945073 WD	219
	220	close(error_pipe[1]);
	221	+ close(name_pipe[0]);
	222	if (f_in != f_out)
	223	close(f_in);
0b32df42	224	+ set_nonblocking(name_pipe[1]);
77945073	225
0b32df42	226	io_start_buffering_out();
77945073 WD	227
	228	set_msg_fd_in(error_pipe[0]);
	229
	230	- generate_files(f_out, flist, local_name);
	231	+ generate_files(f_out, flist, local_name, name_pipe[1]);
	232
	233	get_redo_num(); /* Read final MSG_DONE and any prior messages. */
	234	report(-1);
f6c3b300	235	--- orig/receiver.c 2004-07-16 20:07:22
0b32df42 WD	236	+++ receiver.c 2004-07-17 11:05:36
0b32df42 WD	237	@@ -304,7 +304,8 @@ static int receive_data(int f_in,struct
77945073 WD	238	* main routine for receiver process.
	239	*
	240	* Receiver process runs on the same host as the generator process. */
9be39c35	241	-int recv_files(int f_in, struct file_list flist, char local_name)
77945073 WD	242	+int recv_files(int f_in, struct file_list flist, char local_name,
	243	+ int f_name)
	244	{
	245	int fd1,fd2;
	246	STRUCT_STAT st;
0b32df42	247	@@ -317,7 +318,7 @@ int recv_files(int f_in, struct file_lis
77945073	248	struct file_struct *file;
0b32df42 WD	249	struct stats initial_stats;
	250	int save_make_backups = make_backups;
	251	- int i, recv_ok, phase = 0;
	252	+ int i, len, recv_ok, phase = 0;
	253
	254	if (verbose > 2)
	255	rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
	256	@@ -373,19 +374,25 @@ int recv_files(int f_in, struct file_lis
77945073 WD	257	if (verbose > 2)
	258	rprintf(FINFO,"recv_files(%s)\n",fname);
	259
	260	- fnamecmp = fname;
0b32df42 WD	261	+ len = read_byte(f_name);
	262	+ if (len & 0x80) {
	263	+#if MAXPATHLEN > 32767
	264	+ read_buf(f_name, fnamecmpbuf, 2);
	265	+ len = (len & ~0x80) * 0x10000
	266	+ + fnamecmpbuf[0] * 0x100 + fnamecmpbuf[1];
	267	+#else
	268	+ len = (len & ~0x80) * 0x100 + read_byte(f_name);
	269	+#endif
77945073	270	+ }
0b32df42 WD	271	+ if (len) {
	272	+ read_sbuf(f_name, fnamecmpbuf, len);
	273	+ fnamecmp = fnamecmpbuf;
	274	+ } else
	275	+ fnamecmp = fname;
77945073 WD	276
	277	/* open the file */
	278	fd1 = do_open(fnamecmp, O_RDONLY, 0);
	279
	280	- if (fd1 == -1 && compare_dest != NULL) {
	281	- /* try the file at compare_dest instead */
	282	- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
	283	- compare_dest, fname);
	284	- fnamecmp = fnamecmpbuf;
	285	- fd1 = do_open(fnamecmp, O_RDONLY, 0);
	286	- }
	287	-
	288	if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
	289	rsyserr(FERROR, errno, "fstat %s failed",
	290	full_fname(fnamecmp));