[rsync/rsync-patches.git] / fuzzy.diff

Depends-On-Patch: g2r-basis-filename.diff

The changes to generator.c were greatly simplified, making the patch
easier to maintain and fixing the failing test in the testsuite.
Very lightly tested.

Be sure to run "make proto" before "make".

--- orig/generator.c	2004-07-30 07:18:03
+++ generator.c	2004-07-30 18:11:15
@@ -41,6 +41,7 @@ extern int ignore_times;
 extern int size_only;
 extern int io_timeout;
 extern int protocol_version;
+extern int fuzzy_basis;
 extern int always_checksum;
 extern char *partial_dir;
 extern char *compare_dest;
@@ -239,6 +240,93 @@ static void generate_and_send_sums(int f
 }
 
 
+static void split_names(char *fname, char **dirname, char **basename)
+{
+	char *slash = strrchr(fname, '/');
+	if (slash) {
+		*dirname = fname;
+		*slash = '\0';
+		*basename = slash+1;
+	} else {
+		*basename = fname;
+		*dirname = ".";
+	}
+}
+
+
+static unsigned int measure_name(const char *name, const char *basename,
+				 const char *ext)
+{
+	int namelen = strlen(name);
+	int extlen = strlen(ext);
+	unsigned int score = 0;
+
+	/* Extensions must match */
+	if (namelen <= extlen || strcmp(name + namelen - extlen, ext) != 0)
+		return 0;
+
+	/* Now score depends on similarity of prefix */
+	for (; *name == *basename && *name; name++, basename++)
+		score++;
+	return score;
+}
+
+
+static int find_fuzzy(const char *fname, char *buf, STRUCT_STAT *st_ptr)
+{
+	DIR *d;
+	struct dirent *di;
+	char *basename, *dirname;
+	char mangled_name[MAXPATHLEN];
+	char bestname[MAXPATHLEN];
+	unsigned int bestscore = 0;
+	const char *ext;
+
+	strlcpy(mangled_name, fname, sizeof mangled_name);
+
+	split_names(mangled_name, &dirname, &basename);
+	if (!(d = opendir(dirname))) {
+		rsyserr(FERROR, errno, "recv_generator opendir(%s)", dirname);
+		return -1;
+	}
+
+	/* Get final extension, eg. .gz; never full basename though. */
+	ext = strrchr(basename + 1, '.');
+	if (!ext)
+		ext = basename + strlen(basename); /* ext = "" */
+
+	while ((di = readdir(d)) != NULL) {
+		const char *dname = d_name(di);
+		unsigned int score;
+
+		if (dname[0] == '.' && (dname[1] == '\0'
+		    || (dname[1] == '.' && dname[2] == '\0')))
+			continue;
+
+		score = measure_name(dname, basename, ext);
+		if (verbose > 4) {
+			rprintf(FINFO, "[%s] fuzzy score for %s = %u\n",
+				who_am_i(), dname, score);
+		}
+		if (score > bestscore) {
+			strlcpy(bestname, dname, sizeof bestname);
+			bestscore = score;
+		}
+	}
+	closedir(d);
+
+	/* Found a candidate. */
+	if (bestscore != 0) {
+		pathjoin(buf, MAXPATHLEN, dirname, bestname);
+		if (verbose > 2) {
+			rprintf(FINFO, "[%s] fuzzy match %s->%s\n",
+				who_am_i(), fname, buf);
+		}
+		return link_stat(buf, st_ptr, 0);
+	}
+	return -1;
+}
+
 
 /*
  * Acts on file number @p i from @p flist, whose name is @p fname.
@@ -447,6 +535,15 @@ static void recv_generator(char *fname, 
 		stat_errno = ENOENT;
 	}
 
+	if (statret == -1 && fuzzy_basis) {
+		if (find_fuzzy(fname, fnamecmpbuf, &st) == 0
+		    && S_ISREG(st.st_mode)) {
+			statret = 0;
+			fnamecmp = fnamecmpbuf;
+			fnamecmp_type = G2R_FUZZY;
+		}
+	}
+
 	if (statret == -1) {
 		if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 			return;
@@ -473,7 +570,7 @@ static void recv_generator(char *fname, 
 		return;
 	}
 
-	if (skip_file(fnamecmp, file, &st)) {
+	if (fnamecmp_type != G2R_FUZZY && skip_file(fnamecmp, file, &st)) {
 		if (fnamecmp_type == G2R_FNAME)
 			set_perms(fname, file, &st, PERMS_REPORT);
 		return;
@@ -521,6 +618,21 @@ static void recv_generator(char *fname, 
 notify_others:
 	if (f_out_name >= 0) {
 		write_byte(f_out_name, fnamecmp_type);
+		if (fnamecmp_type == G2R_FUZZY) {
+			uchar lenbuf[3], *lb = lenbuf;
+			int len = strlen(fnamecmpbuf);
+			if (len > 0x7F) {
+#if MAXPATHLEN > 0x7FFF
+				*lb++ = len / 0x10000 + 0x80;
+				*lb++ = len / 0x100;
+#else
+				*lb++ = len / 0x100 + 0x80;
+#endif
+			}
+			*lb = len;
+			write_buf(f_out_name, lenbuf, lb - lenbuf + 1);
+			write_buf(f_out_name, fnamecmpbuf, len);
+		}
 		io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */
 	}
 
--- orig/main.c	2004-07-22 00:10:43
+++ main.c	2004-07-22 00:32:31
@@ -48,6 +48,7 @@ extern int keep_dirlinks;
 extern int preserve_hard_links;
 extern int protocol_version;
 extern int recurse;
+extern int fuzzy_basis;
 extern int relative_paths;
 extern int rsync_port;
 extern int whole_file;
@@ -459,7 +460,7 @@ static int do_recv(int f_in,int f_out,st
 	int pid;
 	int status = 0;
 	int error_pipe[2], name_pipe[2];
-	BOOL need_name_pipe = compare_dest && !dry_run;
+	BOOL need_name_pipe = (compare_dest || fuzzy_basis) && !dry_run;
 
 	/* The receiving side mustn't obey this, or an existing symlink that
 	 * points to an identical file won't be replaced by the referent. */
--- orig/options.c	2004-08-03 15:41:32
+++ options.c	2004-07-16 20:14:12
@@ -86,6 +86,7 @@ int safe_symlinks = 0;
 int copy_unsafe_links = 0;
 int size_only = 0;
 int bwlimit = 0;
+int fuzzy_basis = 0;
 size_t bwlimit_writemax = 0;
 int delete_after = 0;
 int only_existing = 0;
@@ -280,6 +281,7 @@ void usage(enum logcode F)
   rprintf(F," -T, --temp-dir=DIR          create temporary files in directory DIR\n");
   rprintf(F,"     --compare-dest=DIR      also compare destination files relative to DIR\n");
   rprintf(F,"     --link-dest=DIR         create hardlinks to DIR for unchanged files\n");
+  rprintf(F,"     --fuzzy                 use similar file as basis if basis doesn't exist\n");
   rprintf(F," -P                          equivalent to --partial --progress\n");
   rprintf(F," -z, --compress              compress file data\n");
   rprintf(F," -C, --cvs-exclude           auto ignore files in the same way CVS does\n");
@@ -379,6 +381,7 @@ static struct poptOption long_options[] 
   {"temp-dir",        'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
   {"compare-dest",     0,  POPT_ARG_STRING, &compare_dest, 0, 0, 0 },
   {"link-dest",        0,  POPT_ARG_STRING, &compare_dest,  OPT_LINK_DEST, 0, 0 },
+  {"fuzzy",            0,  POPT_ARG_NONE,   &fuzzy_basis, 0, 0, 0 },
   /* TODO: Should this take an optional int giving the compression level? */
   {"compress",        'z', POPT_ARG_NONE,   &do_compression, 0, 0, 0 },
   {"daemon",           0,  POPT_ARG_NONE,   &daemon_opt, 0, 0, 0 },
@@ -1088,6 +1091,9 @@ void server_options(char **args,int *arg
 		}
 	}
 
+	if (fuzzy_basis && am_sender)
+		args[ac++] = "--fuzzy";
+
 	*argc = ac;
 	return;
 
--- orig/receiver.c	2004-07-30 07:17:48
+++ receiver.c	2004-07-30 18:21:38
@@ -323,6 +323,27 @@ static int receive_data(int f_in, char *
 }
 
 
+static void read_gen_name(int fd, char *buf)
+{
+	int len = read_byte(fd);
+	if (len & 0x80) {
+#if MAXPATHLEN > 32767
+		uchar lenbuf[2];
+		read_buf(fd, (char *)lenbuf, 2);
+		len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
+#else
+		len = (len & ~0x80) * 0x100 + read_byte(fd);
+#endif
+	}
+	if (len >= MAXPATHLEN) {
+		rprintf(FERROR, "bogus data on generator name pipe\n");
+		exit_cleanup(RERR_PROTOCOL);
+	}
+
+	read_sbuf(fd, buf, len);
+}
+
+
 static void discard_receive_data(int f_in, OFF_T length)
 {
 	receive_data(f_in, NULL, -1, 0, NULL, -1, length);
@@ -449,6 +470,10 @@ int recv_files(int f_in, struct file_lis
 			case G2R_PARTIAL_DIR:
 				fnamecmp = partialptr ? partialptr : fname;
 				break;
+			case G2R_FUZZY:
+				read_gen_name(f_in_name, fnamecmpbuf);
+				fnamecmp = fnamecmpbuf;
+				break;
 			default:
 				pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 					 compare_dest, fname);
--- orig/rsync.h	2004-07-30 07:16:38
+++ rsync.h	2004-07-30 07:45:40
@@ -120,6 +120,7 @@
 
 #define G2R_FNAME	0x80
 #define G2R_PARTIAL_DIR	0x81
+#define G2R_FUZZY	0x82
 
 
 /* Log-message categories.  FLOG is only used on the daemon side to
--- orig/rsync.yo	2004-08-03 15:34:32
+++ rsync.yo	2004-07-03 19:27:25
@@ -327,6 +327,7 @@ verb(
  -T  --temp-dir=DIR          create temporary files in directory DIR
      --compare-dest=DIR      also compare received files relative to DIR
      --link-dest=DIR         create hardlinks to DIR for unchanged files
+     --fuzzy                 use similar file as basis if basis is gone
  -P                          equivalent to --partial --progress
  -z, --compress              compress file data
  -C, --cvs-exclude           auto ignore files in the same way CVS does
Commit	Line	Data
824abc86	1	Depends-On-Patch: g2r-basis-filename.diff
8c5b8235	2
824abc86	3	The changes to generator.c were greatly simplified, making the patch
8c5b8235	4	easier to maintain and fixing the failing test in the testsuite.
58118c25	5	Very lightly tested.
241013b4	6
824abc86 WD	7	Be sure to run "make proto" before "make".
824abc86 WD	8
09fb2223 WD	9	--- orig/generator.c 2004-07-30 07:18:03
09fb2223 WD	10	+++ generator.c 2004-07-30 18:11:15
58118c25 WD	11	@@ -41,6 +41,7 @@ extern int ignore_times;
	12	extern int size_only;
	13	extern int io_timeout;
	14	extern int protocol_version;
09fb2223	15	+extern int fuzzy_basis;
58118c25	16	extern int always_checksum;
b952a177	17	extern char *partial_dir;
58118c25	18	extern char *compare_dest;
d5753a22	19	@@ -239,6 +240,93 @@ static void generate_and_send_sums(int f
58118c25 WD	20	}
	21
	22
47dd7a31 WD	23	+static void split_names(char fname, char dirname, char *basename)
	24	+{
	25	+ char *slash = strrchr(fname, '/');
	26	+ if (slash) {
	27	+ *dirname = fname;
	28	+ *slash = '\0';
	29	+ *basename = slash+1;
	30	+ } else {
	31	+ *basename = fname;
	32	+ *dirname = ".";
	33	+ }
	34	+}
	35	+
58118c25	36	+
47dd7a31 WD	37	+static unsigned int measure_name(const char name, const char basename,
	38	+ const char *ext)
	39	+{
	40	+ int namelen = strlen(name);
	41	+ int extlen = strlen(ext);
	42	+ unsigned int score = 0;
	43	+
	44	+ /* Extensions must match */
	45	+ if (namelen <= extlen \|\| strcmp(name + namelen - extlen, ext) != 0)
	46	+ return 0;
	47	+
	48	+ /* Now score depends on similarity of prefix */
	49	+ for (; name == basename && *name; name++, basename++)
	50	+ score++;
	51	+ return score;
	52	+}
	53	+
58118c25	54	+
09fb2223	55	+static int find_fuzzy(const char fname, char buf, STRUCT_STAT *st_ptr)
47dd7a31 WD	56	+{
	57	+ DIR *d;
	58	+ struct dirent *di;
	59	+ char basename, dirname;
	60	+ char mangled_name[MAXPATHLEN];
	61	+ char bestname[MAXPATHLEN];
	62	+ unsigned int bestscore = 0;
	63	+ const char *ext;
	64	+
09fb2223	65	+ strlcpy(mangled_name, fname, sizeof mangled_name);
47dd7a31 WD	66	+
	67	+ split_names(mangled_name, &dirname, &basename);
	68	+ if (!(d = opendir(dirname))) {
	69	+ rsyserr(FERROR, errno, "recv_generator opendir(%s)", dirname);
	70	+ return -1;
	71	+ }
	72	+
	73	+ /* Get final extension, eg. .gz; never full basename though. */
	74	+ ext = strrchr(basename + 1, '.');
	75	+ if (!ext)
	76	+ ext = basename + strlen(basename); /* ext = "" */
	77	+
	78	+ while ((di = readdir(d)) != NULL) {
	79	+ const char *dname = d_name(di);
	80	+ unsigned int score;
	81	+
	82	+ if (dname[0] == '.' && (dname[1] == '\0'
	83	+ \|\| (dname[1] == '.' && dname[2] == '\0')))
	84	+ continue;
	85	+
	86	+ score = measure_name(dname, basename, ext);
	87	+ if (verbose > 4) {
8c5b8235 WD	88	+ rprintf(FINFO, "[%s] fuzzy score for %s = %u\n",
8c5b8235 WD	89	+ who_am_i(), dname, score);
47dd7a31 WD	90	+ }
47dd7a31 WD	91	+ if (score > bestscore) {
8c5b8235	92	+ strlcpy(bestname, dname, sizeof bestname);
47dd7a31 WD	93	+ bestscore = score;
	94	+ }
	95	+ }
	96	+ closedir(d);
	97	+
	98	+ /* Found a candidate. */
	99	+ if (bestscore != 0) {
8c5b8235	100	+ pathjoin(buf, MAXPATHLEN, dirname, bestname);
47dd7a31	101	+ if (verbose > 2) {
8c5b8235	102	+ rprintf(FINFO, "[%s] fuzzy match %s->%s\n",
09fb2223	103	+ who_am_i(), fname, buf);
47dd7a31	104	+ }
58118c25	105	+ return link_stat(buf, st_ptr, 0);
47dd7a31 WD	106	+ }
	107	+ return -1;
	108	+}
58118c25 WD	109	+
	110
	111	/*
	112	* Acts on file number @p i from @p flist, whose name is @p fname.
d5753a22	113	@@ -447,6 +535,15 @@ static void recv_generator(char *fname,
09fb2223 WD	114	stat_errno = ENOENT;
09fb2223 WD	115	}
824abc86	116
09fb2223 WD	117	+ if (statret == -1 && fuzzy_basis) {
	118	+ if (find_fuzzy(fname, fnamecmpbuf, &st) == 0
	119	+ && S_ISREG(st.st_mode)) {
	120	+ statret = 0;
	121	+ fnamecmp = fnamecmpbuf;
	122	+ fnamecmp_type = G2R_FUZZY;
	123	+ }
824abc86 WD	124	+ }
824abc86 WD	125	+
09fb2223 WD	126	if (statret == -1) {
09fb2223 WD	127	if (preserve_hard_links && hard_link_check(file, HL_SKIP))
7628f156	128	return;
d5753a22	129	@@ -473,7 +570,7 @@ static void recv_generator(char *fname,
241013b4 MP	130	return;
	131	}
	132
ec8473c6	133	- if (skip_file(fnamecmp, file, &st)) {
09fb2223 WD	134	+ if (fnamecmp_type != G2R_FUZZY && skip_file(fnamecmp, file, &st)) {
09fb2223 WD	135	if (fnamecmp_type == G2R_FNAME)
8c5b8235 WD	136	set_perms(fname, file, &st, PERMS_REPORT);
8c5b8235 WD	137	return;
d5753a22	138	@@ -521,6 +618,21 @@ static void recv_generator(char *fname,
09fb2223 WD	139	notify_others:
	140	if (f_out_name >= 0) {
	141	write_byte(f_out_name, fnamecmp_type);
	142	+ if (fnamecmp_type == G2R_FUZZY) {
	143	+ uchar lenbuf[3], *lb = lenbuf;
	144	+ int len = strlen(fnamecmpbuf);
	145	+ if (len > 0x7F) {
	146	+#if MAXPATHLEN > 0x7FFF
	147	+ *lb++ = len / 0x10000 + 0x80;
	148	+ *lb++ = len / 0x100;
	149	+#else
	150	+ *lb++ = len / 0x100 + 0x80;
	151	+#endif
	152	+ }
	153	+ *lb = len;
	154	+ write_buf(f_out_name, lenbuf, lb - lenbuf + 1);
	155	+ write_buf(f_out_name, fnamecmpbuf, len);
	156	+ }
	157	io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */
	158	}
	159
2f5fa77e WD	160	--- orig/main.c 2004-07-22 00:10:43
2f5fa77e WD	161	+++ main.c 2004-07-22 00:32:31
d5753a22	162	@@ -48,6 +48,7 @@ extern int keep_dirlinks;
495f1899 WD	163	extern int preserve_hard_links;
	164	extern int protocol_version;
	165	extern int recurse;
09fb2223	166	+extern int fuzzy_basis;
495f1899 WD	167	extern int relative_paths;
	168	extern int rsync_port;
	169	extern int whole_file;
d5753a22	170	@@ -459,7 +460,7 @@ static int do_recv(int f_in,int f_out,st
495f1899 WD	171	int pid;
	172	int status = 0;
	173	int error_pipe[2], name_pipe[2];
c2530f70	174	- BOOL need_name_pipe = compare_dest && !dry_run;
09fb2223	175	+ BOOL need_name_pipe = (compare_dest \|\| fuzzy_basis) && !dry_run;
495f1899	176
d5753a22 WD	177	/* The receiving side mustn't obey this, or an existing symlink that
	178	* points to an identical file won't be replaced by the referent. */
	179	--- orig/options.c 2004-08-03 15:41:32
f6c3b300	180	+++ options.c 2004-07-16 20:14:12
d5753a22	181	@@ -86,6 +86,7 @@ int safe_symlinks = 0;
f6c3b300 WD	182	int copy_unsafe_links = 0;
	183	int size_only = 0;
	184	int bwlimit = 0;
09fb2223	185	+int fuzzy_basis = 0;
f6c3b300 WD	186	size_t bwlimit_writemax = 0;
	187	int delete_after = 0;
	188	int only_existing = 0;
d5753a22	189	@@ -280,6 +281,7 @@ void usage(enum logcode F)
c2530f70	190	rprintf(F," -T, --temp-dir=DIR create temporary files in directory DIR\n");
f0533c4c WD	191	rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n");
	192	rprintf(F," --link-dest=DIR create hardlinks to DIR for unchanged files\n");
	193	+ rprintf(F," --fuzzy use similar file as basis if basis doesn't exist\n");
	194	rprintf(F," -P equivalent to --partial --progress\n");
	195	rprintf(F," -z, --compress compress file data\n");
	196	rprintf(F," -C, --cvs-exclude auto ignore files in the same way CVS does\n");
d5753a22	197	@@ -379,6 +381,7 @@ static struct poptOption long_options[]
f0533c4c WD	198	{"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
	199	{"compare-dest", 0, POPT_ARG_STRING, &compare_dest, 0, 0, 0 },
	200	{"link-dest", 0, POPT_ARG_STRING, &compare_dest, OPT_LINK_DEST, 0, 0 },
09fb2223	201	+ {"fuzzy", 0, POPT_ARG_NONE, &fuzzy_basis, 0, 0, 0 },
f0533c4c WD	202	/* TODO: Should this take an optional int giving the compression level? */
	203	{"compress", 'z', POPT_ARG_NONE, &do_compression, 0, 0, 0 },
	204	{"daemon", 0, POPT_ARG_NONE, &daemon_opt, 0, 0, 0 },
d5753a22	205	@@ -1088,6 +1091,9 @@ void server_options(char *args,int arg
f74d2272	206	}
241013b4	207	}
7b675ff5	208
09fb2223	209	+ if (fuzzy_basis && am_sender)
241013b4	210	+ args[ac++] = "--fuzzy";
7b675ff5	211	+
241013b4	212	*argc = ac;
f74d2272	213	return;
7b675ff5	214
09fb2223 WD	215	--- orig/receiver.c 2004-07-30 07:17:48
	216	+++ receiver.c 2004-07-30 18:21:38
	217	@@ -323,6 +323,27 @@ static int receive_data(int f_in, char *
	218	}
	219
	220
	221	+static void read_gen_name(int fd, char *buf)
	222	+{
	223	+ int len = read_byte(fd);
	224	+ if (len & 0x80) {
	225	+#if MAXPATHLEN > 32767
	226	+ uchar lenbuf[2];
	227	+ read_buf(fd, (char *)lenbuf, 2);
	228	+ len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
	229	+#else
	230	+ len = (len & ~0x80) * 0x100 + read_byte(fd);
	231	+#endif
	232	+ }
	233	+ if (len >= MAXPATHLEN) {
	234	+ rprintf(FERROR, "bogus data on generator name pipe\n");
	235	+ exit_cleanup(RERR_PROTOCOL);
	236	+ }
	237	+
	238	+ read_sbuf(fd, buf, len);
	239	+}
	240	+
	241	+
	242	static void discard_receive_data(int f_in, OFF_T length)
	243	{
	244	receive_data(f_in, NULL, -1, 0, NULL, -1, length);
	245	@@ -449,6 +470,10 @@ int recv_files(int f_in, struct file_lis
	246	case G2R_PARTIAL_DIR:
	247	fnamecmp = partialptr ? partialptr : fname;
	248	break;
	249	+ case G2R_FUZZY:
	250	+ read_gen_name(f_in_name, fnamecmpbuf);
	251	+ fnamecmp = fnamecmpbuf;
	252	+ break;
	253	default:
	254	pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
	255	compare_dest, fname);
	256	--- orig/rsync.h 2004-07-30 07:16:38
	257	+++ rsync.h 2004-07-30 07:45:40
	258	@@ -120,6 +120,7 @@
	259
	260	#define G2R_FNAME 0x80
	261	#define G2R_PARTIAL_DIR 0x81
	262	+#define G2R_FUZZY 0x82
	263
	264
	265	/* Log-message categories. FLOG is only used on the daemon side to
d5753a22	266	--- orig/rsync.yo 2004-08-03 15:34:32
824abc86	267	+++ rsync.yo 2004-07-03 19:27:25
b952a177	268	@@ -327,6 +327,7 @@ verb(
f0533c4c WD	269	-T --temp-dir=DIR create temporary files in directory DIR
	270	--compare-dest=DIR also compare received files relative to DIR
	271	--link-dest=DIR create hardlinks to DIR for unchanged files
	272	+ --fuzzy use similar file as basis if basis is gone
	273	-P equivalent to --partial --progress
	274	-z, --compress compress file data
	275	-C, --cvs-exclude auto ignore files in the same way CVS does