X-Git-Url: https://mattmccutchen.net/rsync/rsync-patches.git/blobdiff_plain/f0533c4cc035bfc6c614f9a59ba5a651daf5ba0f..0f6260347bf6c29859ed4808f04222f84c2a8817:/fuzzy.diff diff --git a/fuzzy.diff b/fuzzy.diff index 3568814..73d6b74 100644 --- a/fuzzy.diff +++ b/fuzzy.diff @@ -1,366 +1,280 @@ -From: Rusty Russell -Date: Wed, 03 Apr 2002 17:08:57 +1000 +The changes to generator.c were greatly simplified, making the patch +easier to maintain and fixing the failing test in the testsuite. +Lightly tested. -Found old patch on google, and updated it for 2.5.4 (I know, but that's what -apt-get source gave me). +Be sure to run "make proto" before "make". -Compiles, otherwise untested. -Rusty. - -[Updated for current CVS version by Wayne Davison. Passes *MOST* of the -test suite, but otherwise UNTESTED.] --- - Anyone who quotes me in their sig is an idiot. -- Rusty Russell. - ---- Makefile.in 15 May 2004 00:48:11 -0000 1.101 -+++ Makefile.in 21 May 2004 09:51:19 -0000 -@@ -32,7 +32,7 @@ ZLIBOBJ=zlib/deflate.o zlib/infblock.o z - zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \ - zlib/zutil.o zlib/adler32.o - OBJS1=rsync.o generator.o receiver.o cleanup.o sender.o exclude.o util.o \ -- main.o checksum.o match.o syscall.o log.o backup.o -+ main.o checksum.o match.o syscall.o log.o backup.o alternate.o - OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o \ - fileio.o batch.o clientname.o - OBJS3=progress.o pipe.o ---- generator.c 18 May 2004 08:50:17 -0000 1.85 -+++ generator.c 21 May 2004 09:51:19 -0000 -@@ -51,6 +51,7 @@ extern int list_only; - extern int only_existing; - extern int orig_umask; - extern int safe_symlinks; -+extern int fuzzy; - - extern struct exclude_list_struct server_exclude_list; - -@@ -258,7 +259,61 @@ static void generate_and_send_sums(struc - } +--- orig/generator.c 2005-01-17 23:11:45 ++++ generator.c 2005-01-16 02:16:38 +@@ -44,6 +44,7 @@ extern int size_only; + extern OFF_T max_size; + extern int io_timeout; + extern int protocol_version; ++extern int fuzzy_basis; + extern int always_checksum; + extern char *partial_dir; + extern char *basis_dir[]; +@@ -242,6 +243,83 @@ static void generate_and_send_sums(int f } -+/* Returns -1 for can't open (null file), -2 for skip */ -+static int open_base_file(struct file_struct *file, char *fname, int statret, -+ STRUCT_STAT *st) + ++static unsigned int measure_name(const char *name, const char *basename, ++ const char *ext) +{ -+ int fd = -1; ++ int namelen = strlen(name); ++ int extlen = strlen(ext); ++ unsigned int score = 0; + -+ if (statret == 0) { -+ if (S_ISREG(st->st_mode)) { -+ if (update_only -+ && cmp_modtime(st->st_mtime, file->modtime) > 0) { -+ if (verbose > 1) -+ rprintf(FINFO, "%s is newer\n", fname); -+ return -2; -+ } -+ if (skip_file(fname, file, st)) { -+ set_perms(fname, file, st, PERMS_REPORT); -+ return -2; -+ } -+ fd = do_open(fname, O_RDONLY, 0); -+ if (fd == -1) { -+ rsyserr(FERROR, errno, "failed to open %s, continuing", -+ full_fname(fname)); -+ return -1; -+ } -+ return fd; -+ } else { -+ /* Try to use symlink contents */ -+ if (S_ISLNK(st->st_mode)) { -+ fd = do_open_regular(fname); -+ /* Don't delete yet; receiver will need it */ -+ } else { -+ if (delete_file(fname) != 0) { -+ if (fd != -1) -+ close(fd); -+ return -2; -+ } -+ } -+ } ++ /* Extensions must match */ ++ if (namelen <= extlen || strcmp(name + namelen - extlen, ext) != 0) ++ return 0; ++ ++ /* Now score depends on similarity of prefix */ ++ for (; *name == *basename && *name; name++, basename++) ++ score++; ++ return score; ++} ++ ++ ++static int find_fuzzy(const char *fname, char *buf, STRUCT_STAT *st_ptr) ++{ ++ DIR *d; ++ struct dirent *di; ++ char *basename, *dirname, *slash; ++ char bestname[MAXPATHLEN]; ++ unsigned int bestscore = 0; ++ const char *ext; ++ ++ strlcpy(buf, fname, MAXPATHLEN); ++ if ((slash = strrchr(buf, '/')) != NULL) { ++ dirname = buf; ++ *slash = '\0'; ++ basename = slash + 1; ++ } else { ++ basename = buf; ++ dirname = "."; ++ } ++ ++ if (!(d = opendir(dirname))) { ++ rsyserr(FERROR, errno, "recv_generator opendir(%s)", dirname); ++ return -1; + } + -+ if (fd == -1 && compare_dest != NULL) -+ fd = open_alternate_base_comparedir(fname); ++ /* Get final extension, eg. .gz; never full basename though. */ ++ for (ext = basename; *ext == '.'; ext++) {} ++ if (!(ext = strrchr(ext, '.'))) ++ ext = basename + strlen(basename); /* ext = "" */ ++ ++ while ((di = readdir(d)) != NULL) { ++ const char *dname = d_name(di); ++ unsigned int score; + -+ if (fd == -1 && fuzzy) -+ fd = open_alternate_base_fuzzy(fname); ++ if (dname[0] == '.' && (dname[1] == '\0' ++ || (dname[1] == '.' && dname[2] == '\0'))) ++ continue; + -+ /* Update stat to understand size */ -+ if (fd != -1) { -+ if (do_fstat(fd, st) != 0) { -+ rsyserr(FERROR, errno, "fstat %s", full_fname(fname)); ++ score = measure_name(dname, basename, ext); ++ if (verbose > 4) { ++ rprintf(FINFO, "fuzzy score for %s = %u\n", ++ dname, score); ++ } ++ if (score > bestscore) { ++ strlcpy(bestname, dname, sizeof bestname); ++ bestscore = score; + } + } - -+ return fd; ++ closedir(d); ++ ++ /* Found a candidate. */ ++ if (bestscore != 0) { ++ strlcpy(basename, MAXPATHLEN - (basename - buf), bestname); ++ if (verbose > 2) ++ rprintf(FINFO, "fuzzy match %s->%s\n", fname, buf); ++ return link_stat(buf, st_ptr, 0); ++ } ++ return -1; +} ++ - /** + /* * Acts on file number @p i from @p flist, whose name is @p fname. -@@ -274,8 +329,6 @@ void recv_generator(char *fname, struct - STRUCT_STAT st; - struct map_struct *mapbuf; - int statret; -- char *fnamecmp; -- char fnamecmpbuf[MAXPATHLEN]; - - if (list_only) - return; -@@ -413,107 +466,39 @@ void recv_generator(char *fname, struct - } - #endif - -- if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER)) -- return; -- -- if (!S_ISREG(file->mode)) { -- rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname); -- return; -- } -- -- fnamecmp = fname; -- -- if (statret == -1 && compare_dest != NULL) { -- /* try the file at compare_dest instead */ -- int saveerrno = errno; -- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname); -- statret = link_stat(fnamecmpbuf,&st); -- if (!S_ISREG(st.st_mode)) -- statret = -1; -- if (statret == -1) -- errno = saveerrno; --#if HAVE_LINK -- else if (link_dest && !dry_run) { -- if (do_link(fnamecmpbuf, fname) != 0) { -- if (verbose > 0) { -- rsyserr(FINFO, errno, "link %s => %s", -- fnamecmpbuf, fname); -- } -- } -- fnamecmp = fnamecmpbuf; -- } --#endif -- else -- fnamecmp = fnamecmpbuf; -- } -- -- if (statret == -1) { -- if (preserve_hard_links && hard_link_check(file, HL_SKIP)) -- return; -- if (errno == ENOENT) { -- write_int(f_out,i); -- if (!dry_run) write_sum_head(f_out, NULL); -- } else if (verbose > 1) { -+ /* Failed to stat for some reason besides "not found". */ -+ if (statret == -1 && errno != ENOENT) { -+ if (verbose > 1) { - rsyserr(FERROR, errno, -- "recv_generator: failed to open %s", -+ "recv_generator failed to stat %s", - full_fname(fname)); - } - return; - } - -- if (!S_ISREG(st.st_mode)) { -- if (delete_file(fname) != 0) { -- return; -- } -- -- /* now pretend the file didn't exist */ -- if (preserve_hard_links && hard_link_check(file, HL_SKIP)) -- return; -- write_int(f_out,i); -- if (!dry_run) write_sum_head(f_out, NULL); -- return; -- } -- -- if (opt_ignore_existing && fnamecmp == fname) { -- if (verbose > 1) -- rprintf(FINFO,"%s exists\n",fname); -- return; -- } -- -- if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) { -- if (verbose > 1) -- rprintf(FINFO,"%s is newer\n",fname); -- return; -- } -- -- if (skip_file(fname, file, &st)) { -- if (fnamecmp == fname) -- set_perms(fname, file, &st, PERMS_REPORT); -- return; -- } -- -- if (dry_run) { -- write_int(f_out,i); -+ if ((fd = open_base_file(file, fname, statret, &st)) == -2) - return; -- } - -- if (disable_deltas_p()) { -- write_int(f_out,i); -- write_sum_head(f_out, NULL); -- return; -+ if ((disable_deltas_p() || dry_run) && fd != -1) { -+ close(fd); -+ fd = -1; - } +@@ -496,6 +574,15 @@ static void recv_generator(char *fname, + } else + partialptr = NULL; -- /* open the file */ -- fd = do_open(fnamecmp, O_RDONLY, 0); -- - if (fd == -1) { -- rsyserr(FERROR, errno, "failed to open %s, continuing", -- full_fname(fnamecmp)); -- /* pretend the file didn't exist */ -+ /* the file didn't exist, or we can pretend it doesn't */ ++ if (statret == -1 && fuzzy_basis) { ++ if (find_fuzzy(fname, fnamecmpbuf, &st) == 0 ++ && S_ISREG(st.st_mode)) { ++ statret = 0; ++ fnamecmp = fnamecmpbuf; ++ fnamecmp_type = FNAMECMP_FUZZY; ++ } ++ } ++ + if (statret == -1) { if (preserve_hard_links && hard_link_check(file, HL_SKIP)) return; -- write_int(f_out,i); -- write_sum_head(f_out, NULL); -+ write_int(f_out, i); -+ if (!dry_run) -+ write_sum_head(f_out, NULL); -+ return; -+ } -+ -+ if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER)) -+ return; -+ -+ if (!S_ISREG(file->mode)) { -+ rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname); - return; - } - -@@ -523,7 +508,7 @@ void recv_generator(char *fname, struct - mapbuf = NULL; - - if (verbose > 3) { -- rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp, -+ rprintf(FINFO, "gen mapped %s of size %.0f\n", fname, - (double)st.st_size); - } +@@ -524,6 +611,8 @@ static void recv_generator(char *fname, ---- options.c 21 May 2004 09:44:32 -0000 1.150 -+++ options.c 21 May 2004 09:51:20 -0000 -@@ -91,6 +91,7 @@ int ignore_errors = 0; - int modify_window = 0; - int blocking_io = -1; - int checksum_seed = 0; -+int fuzzy = 0; - unsigned int block_size = 0; + if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH) + ; ++ else if (fnamecmp_type == FNAMECMP_FUZZY) ++ ; + else if (unchanged_file(fnamecmp, file, &st)) { + if (fnamecmp_type == FNAMECMP_FNAME) + set_perms(fname, file, &st, PERMS_REPORT); +@@ -598,8 +687,24 @@ notify_others: + write_int(f_out, i); + if (protocol_version >= 29 && inplace && !read_batch) + write_byte(f_out, fnamecmp_type); +- if (f_out_name >= 0) ++ if (f_out_name >= 0) { + write_byte(f_out_name, fnamecmp_type); ++ if (fnamecmp_type == FNAMECMP_FUZZY) { ++ uchar lenbuf[3], *lb = lenbuf; ++ int len = strlen(fnamecmpbuf); ++ if (len > 0x7F) { ++#if MAXPATHLEN > 0x7FFF ++ *lb++ = len / 0x10000 + 0x80; ++ *lb++ = len / 0x100; ++#else ++ *lb++ = len / 0x100 + 0x80; ++#endif ++ } ++ *lb = len; ++ write_buf(f_out_name, lenbuf, lb - lenbuf + 1); ++ write_buf(f_out_name, fnamecmpbuf, len); ++ } ++ } + if (dry_run || read_batch) + return; +--- orig/main.c 2005-01-17 23:11:45 ++++ main.c 2005-01-14 18:33:15 +@@ -48,6 +48,7 @@ extern int keep_dirlinks; + extern int preserve_hard_links; + extern int protocol_version; + extern int recurse; ++extern int fuzzy_basis; + extern int relative_paths; + extern int rsync_port; + extern int whole_file; +@@ -464,7 +465,8 @@ static int do_recv(int f_in,int f_out,st + int pid; + int status = 0; + int error_pipe[2], name_pipe[2]; +- BOOL need_name_pipe = (basis_dir[0] || partial_dir) && !dry_run; ++ BOOL need_name_pipe = (basis_dir[0] || partial_dir || fuzzy_basis) ++ && !dry_run; -@@ -266,6 +267,7 @@ void usage(enum logcode F) - rprintf(F," -T --temp-dir=DIR create temporary files in directory DIR\n"); + /* The receiving side mustn't obey this, or an existing symlink that + * points to an identical file won't be replaced by the referent. */ +--- orig/options.c 2005-01-17 23:11:45 ++++ options.c 2005-01-15 21:08:13 +@@ -86,6 +86,7 @@ int copy_unsafe_links = 0; + int size_only = 0; + int daemon_bwlimit = 0; + int bwlimit = 0; ++int fuzzy_basis = 0; + size_t bwlimit_writemax = 0; + int delete_after = 0; + int only_existing = 0; +@@ -288,6 +289,7 @@ void usage(enum logcode F) rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n"); - rprintf(F," --link-dest=DIR create hardlinks to DIR for unchanged files\n"); -+ rprintf(F," --fuzzy use similar file as basis if basis doesn't exist\n"); + rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n"); + rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n"); ++ rprintf(F," --fuzzy find similar file for basis when no dest file\n"); rprintf(F," -P equivalent to --partial --progress\n"); rprintf(F," -z, --compress compress file data\n"); rprintf(F," -C, --cvs-exclude auto ignore files in the same way CVS does\n"); -@@ -363,6 +365,7 @@ static struct poptOption long_options[] - {"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 }, - {"compare-dest", 0, POPT_ARG_STRING, &compare_dest, 0, 0, 0 }, - {"link-dest", 0, POPT_ARG_STRING, &compare_dest, OPT_LINK_DEST, 0, 0 }, -+ {"fuzzy", 0, POPT_ARG_NONE, &fuzzy, 0, 0, 0 }, +@@ -384,6 +386,7 @@ static struct poptOption long_options[] + {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, + {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, + {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 }, ++ {"fuzzy", 0, POPT_ARG_NONE, &fuzzy_basis, 0, 0, 0 }, /* TODO: Should this take an optional int giving the compression level? */ {"compress", 'z', POPT_ARG_NONE, &do_compression, 0, 0, 0 }, - {"daemon", 0, POPT_ARG_NONE, &daemon_opt, 0, 0, 0 }, -@@ -972,6 +975,9 @@ void server_options(char **args,int *arg - } + {"stats", 0, POPT_ARG_NONE, &do_stats, 0, 0, 0 }, +@@ -1234,6 +1237,9 @@ void server_options(char **args,int *arg + args[ac++] = "--no-relative"; } -+ if (fuzzy && am_sender) ++ if (fuzzy_basis && am_sender) + args[ac++] = "--fuzzy"; + *argc = ac; return; ---- receiver.c 21 May 2004 08:27:04 -0000 1.79 -+++ receiver.c 21 May 2004 09:51:20 -0000 -@@ -47,6 +47,7 @@ extern int ignore_errors; - extern int orig_umask; - extern int keep_partial; - extern int checksum_seed; -+extern int fuzzy; - - static void delete_one(char *fn, int is_dir) - { -@@ -295,8 +296,6 @@ int recv_files(int f_in,struct file_list - char *fname, fbuf[MAXPATHLEN]; - char template[MAXPATHLEN]; - char fnametmp[MAXPATHLEN]; -- char *fnamecmp; -- char fnamecmpbuf[MAXPATHLEN]; - struct map_struct *mapbuf; - int i; - struct file_struct *file; -@@ -359,35 +358,31 @@ int recv_files(int f_in,struct file_list - if (verbose > 2) - rprintf(FINFO,"recv_files(%s)\n",fname); +--- orig/receiver.c 2005-01-17 23:11:45 ++++ receiver.c 2005-01-15 21:21:02 +@@ -324,6 +324,27 @@ static int receive_data(int f_in, char * + } -- fnamecmp = fname; -- - /* open the file */ -- fd1 = do_open(fnamecmp, O_RDONLY, 0); -+ fd1 = do_open(fname, O_RDONLY, 0); -- if (fd1 == -1 && compare_dest != NULL) { -- /* try the file at compare_dest instead */ -- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, -- compare_dest, fname); -- fnamecmp = fnamecmpbuf; -- fd1 = do_open(fnamecmp, O_RDONLY, 0); -- } -+ if (fd1 == -1 && compare_dest != NULL) -+ fd1 = open_alternate_base_comparedir(fname); ++static void read_gen_name(int fd, char *buf) ++{ ++ int len = read_byte(fd); ++ if (len & 0x80) { ++#if MAXPATHLEN > 32767 ++ uchar lenbuf[2]; ++ read_buf(fd, (char *)lenbuf, 2); ++ len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1]; ++#else ++ len = (len & ~0x80) * 0x100 + read_byte(fd); ++#endif ++ } ++ if (len >= MAXPATHLEN) { ++ rprintf(FERROR, "bogus data on generator name pipe\n"); ++ exit_cleanup(RERR_PROTOCOL); ++ } + -+ if (fd1 == -1 && fuzzy) -+ fd1 = open_alternate_base_fuzzy(fname); - - if (fd1 != -1 && do_fstat(fd1,&st) != 0) { - rsyserr(FERROR, errno, "fstat %s failed", -- full_fname(fnamecmp)); -+ full_fname(fname)); - receive_data(f_in,NULL,-1,NULL,file->length); - close(fd1); - continue; - } - -- if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) { -+ if (fd1 != -1 && S_ISDIR(st.st_mode)) { - /* this special handling for directories - * wouldn't be necessary if robust_rename() - * and the underlying robust_unlink could cope - * with directories - */ - rprintf(FERROR,"recv_files: %s is a directory\n", -- full_fname(fnamecmp)); -+ full_fname(fname)); - receive_data(f_in, NULL, -1, NULL, file->length); - close(fd1); - continue; -@@ -408,8 +403,10 @@ int recv_files(int f_in,struct file_list ++ read_sbuf(fd, buf, len); ++} ++ ++ + static void discard_receive_data(int f_in, OFF_T length) + { + receive_data(f_in, NULL, -1, 0, NULL, -1, length); +@@ -454,6 +475,10 @@ int recv_files(int f_in, struct file_lis + case FNAMECMP_BACKUP: + fnamecmp = get_backup_name(fname); + break; ++ case FNAMECMP_FUZZY: ++ read_gen_name(f_in_name, fnamecmpbuf); ++ fnamecmp = fnamecmpbuf; ++ break; + default: + if (j >= basis_dir_cnt) { + rprintf(FERROR, +--- orig/rsync.h 2005-01-17 23:11:45 ++++ rsync.h 2005-01-15 21:24:09 +@@ -128,6 +128,7 @@ + #define FNAMECMP_FNAME 0x80 + #define FNAMECMP_PARTIAL_DIR 0x81 + #define FNAMECMP_BACKUP 0x82 ++#define FNAMECMP_FUZZY 0x83 - if (fd1 != -1 && st.st_size > 0) { - mapbuf = map_file(fd1,st.st_size); -- if (verbose > 2) -- rprintf(FINFO,"recv mapped %s of size %.0f\n",fnamecmp,(double)st.st_size); -+ if (verbose > 2) { -+ rprintf(FINFO, "recv mapped %s of size %.0f\n", -+ fname, (double)st.st_size); -+ } - } else - mapbuf = NULL; ---- rsync.yo 21 May 2004 09:44:32 -0000 1.170 -+++ rsync.yo 21 May 2004 09:51:21 -0000 -@@ -324,6 +324,7 @@ verb( - -T --temp-dir=DIR create temporary files in directory DIR + /* Log-message categories. FLOG is only used on the daemon side to +--- orig/rsync.yo 2005-01-17 23:11:46 ++++ rsync.yo 2005-01-15 21:48:52 +@@ -358,6 +358,7 @@ verb( --compare-dest=DIR also compare received files relative to DIR - --link-dest=DIR create hardlinks to DIR for unchanged files -+ --fuzzy use similar file as basis if basis is gone + --copy-dest=DIR ... and include copies of unchanged files + --link-dest=DIR hardlink to files in DIR when unchanged ++ --fuzzy find similar file for basis when no dest -P equivalent to --partial --progress -z, --compress compress file data -C, --cvs-exclude auto ignore files in the same way CVS does +@@ -878,6 +879,11 @@ Note that rsync versions prior to 2.6.1 + (or implied by -a). You can work-around this bug by avoiding the -o option + when sending to an old rsync. + ++dit(bf(--fuzzy)) This option tells rsync that it should look around for a ++basis file for any destination file that is missing. The current algorithm ++looks for a similarly-named file in the same directory as the destination ++file, and, if found, uses that to try to speed up the transfer. ++ + dit(bf(-z, --compress)) With this option, rsync compresses any data from + the files that it sends to the destination machine. This + option is useful on slow connections. The compression method used is the