Matt McCutchen's Web Site / rsync / rsync-patches.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Transformed shell script into perl script and improved it to allow
[rsync/rsync-patches.git] / compare-dest.diff
CommitLineData
09add1ae
WD		 1This patch allows multiple --compare-dest or --link-dest options to be
2used, making the transfer of some files more optimal. Note that the
3algorithm does NOT search for the best match -- it stops at the first
4match and uses that as the basis file for the transfer, so be sure to
8dd60649
WD		 5order your arguments appropriately (the args are searched in the order
6they are suppled).
09add1ae 7
8dd60649
WD		 8Before compiling, be sure to run "make proto".
9
8cec1ead
WD		 10--- generator.c 30 Jun 2004 07:27:30 -0000 1.93
11+++ generator.c 30 Jun 2004 07:40:25 -0000
09add1ae
WD		 12@@ -42,7 +42,7 @@ extern int size_only;
13 extern int io_timeout;
14 extern int protocol_version;
15 extern int always_checksum;
16-extern char *compare_dest;
17+extern char *compare_dest[];
18 extern int link_dest;
19 extern int whole_file;
20 extern int local_server;
4c189bcd 21@@ -80,13 +80,12 @@ static int skip_file(char *fname, struct
09add1ae
WD		 22 if (always_checksum && S_ISREG(st->st_mode)) {
23 char sum[MD4_SUM_LENGTH];
24 char fnamecmpdest[MAXPATHLEN];
25+ int i;
26
27- if (compare_dest != NULL) {
28- if (access(fname, 0) != 0) {
29- pathjoin(fnamecmpdest, sizeof fnamecmpdest,
30- compare_dest, fname);
31- fname = fnamecmpdest;
32- }
4c189bcd 33+ for (i = 0; compare_dest[i] != NULL && access(fname, 0) < 0; i++) {
09add1ae
WD		 34+ pathjoin(fnamecmpdest, sizeof fnamecmpdest,
35+ compare_dest[i], fname);
36+ fname = fnamecmpdest;
37 }
38 file_checksum(fname,sum,st->st_size);
39 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
8dd60649
WD		 40@@ -267,7 +266,7 @@ static void generate_and_send_sums(struc
41 * out. It might be wrong.
42 */
43 static void recv_generator(char *fname, struct file_struct *file, int i,
44- int f_out)
45+ int f_out, int f_nameout)
46 {
47 int fd;
48 STRUCT_STAT st;
49@@ -424,15 +423,22 @@ static void recv_generator(char *fname,
09add1ae
WD		 50
51 fnamecmp = fname;
52
53- if (statret == -1 && compare_dest != NULL) {
54+ if (statret == -1 && compare_dest[0] != NULL) {
55 /* try the file at compare_dest instead */
56 int saveerrno = errno;
57- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
58- statret = link_stat(fnamecmpbuf, &st, 0);
59- if (!S_ISREG(st.st_mode))
60- statret = -1;
8dd60649 61- if (statret == -1)
09add1ae
WD		 62+ int i;
63+ for (i = 0; compare_dest[i] != NULL; i++) {
64+ pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest[i], fname);
65+ if ((statret = link_stat(fnamecmpbuf, &st, 0)) == 0) {
66+ if (S_ISREG(st.st_mode))
67+ break;
68+ statret = -1;
69+ }
70+ }
8dd60649 71+ if (statret < 0) {
09add1ae 72 errno = saveerrno;
8dd60649
WD		 73+ *fnamecmpbuf = '\0';
74+ }
09add1ae 75 #if HAVE_LINK
8dd60649
WD		 76 else if (link_dest && !dry_run) {
77 if (do_link(fnamecmpbuf, fname) != 0) {
78@@ -440,18 +446,22 @@ static void recv_generator(char *fname,
79 rsyserr(FINFO, errno, "link %s => %s",
80 fnamecmpbuf, fname);
81 }
82- }
83- fnamecmp = fnamecmpbuf;
84+ fnamecmp = fnamecmpbuf;
85+ } else
86+ *fnamecmpbuf = '\0';
87 }
88 #endif
89 else
90 fnamecmp = fnamecmpbuf;
91- }
92+ } else
93+ *fnamecmpbuf = '\0';
94
95 if (statret == -1) {
96 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
97 return;
98 if (errno == ENOENT) {
99+ if (f_nameout >= 0)
100+ write(f_nameout, "", 1);
101 write_int(f_out,i);
102 if (!dry_run)
103 write_sum_head(f_out, NULL);
104@@ -471,19 +481,21 @@ static void recv_generator(char *fname,
105 /* now pretend the file didn't exist */
106 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
107 return;
108+ if (f_nameout >= 0)
109+ write(f_nameout, "", 1);
110 write_int(f_out,i);
111 if (!dry_run)
112 write_sum_head(f_out, NULL);
113 return;
114 }
115
116- if (opt_ignore_existing && fnamecmp == fname) {
117+ if (opt_ignore_existing && !*fnamecmpbuf) {
118 if (verbose > 1)
119 rprintf(FINFO,"%s exists\n",fname);
120 return;
121 }
122
123- if (update_only && fnamecmp == fname
124+ if (update_only && !*fnamecmpbuf
125 && cmp_modtime(st.st_mtime, file->modtime) > 0) {
126 if (verbose > 1)
127 rprintf(FINFO,"%s is newer\n",fname);
128@@ -491,17 +503,21 @@ static void recv_generator(char *fname,
129 }
130
131 if (skip_file(fname, file, &st)) {
132- if (fnamecmp == fname)
133+ if (!*fnamecmpbuf)
134 set_perms(fname, file, &st, PERMS_REPORT);
135 return;
136 }
137
138 if (dry_run) {
139+ if (f_nameout >= 0)
140+ write(f_nameout, "", 1);
141 write_int(f_out,i);
142 return;
143 }
144
145 if (disable_deltas_p()) {
146+ if (f_nameout >= 0)
147+ write(f_nameout, "", 1);
148 write_int(f_out,i);
149 write_sum_head(f_out, NULL);
150 return;
151@@ -516,6 +532,8 @@ static void recv_generator(char *fname,
152 /* pretend the file didn't exist */
153 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
154 return;
155+ if (f_nameout >= 0)
156+ write(f_nameout, "", 1);
157 write_int(f_out,i);
158 write_sum_head(f_out, NULL);
159 return;
160@@ -534,6 +552,8 @@ static void recv_generator(char *fname,
161 if (verbose > 2)
162 rprintf(FINFO, "generating and sending sums for %d\n", i);
163
164+ if (f_nameout >= 0)
165+ write(f_nameout, fnamecmpbuf, strlen(fnamecmpbuf) + 1);
166 write_int(f_out,i);
167 generate_and_send_sums(mapbuf, st.st_size, f_out);
168
8cec1ead 169@@ -543,7 +563,8 @@ static void recv_generator(char *fname,
8dd60649
WD		 170 }
171
172
173-void generate_files(int f, struct file_list *flist, char *local_name)
174+void generate_files(int f, struct file_list *flist, char *local_name,
175+ int f_nameout)
176 {
177 int i;
8cec1ead 178 int phase = 0;
8dd60649
WD		 179@@ -584,7 +605,7 @@ void generate_files(int f, struct file_l
180 }
181
182 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
183- file, i, f);
184+ file, i, f, f_nameout);
185 }
186
187 phase++;
188@@ -601,7 +622,7 @@ void generate_files(int f, struct file_l
189 while ((i = get_redo_num()) != -1) {
190 struct file_struct *file = flist->files[i];
191 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
192- file, i, f);
193+ file, i, f, f_nameout);
194 }
195
196 phase++;
197@@ -620,7 +641,7 @@ void generate_files(int f, struct file_l
198 if (!file->basename || !S_ISDIR(file->mode))
199 continue;
200 recv_generator(local_name ? local_name : f_name(file),
201- file, i, -1);
202+ file, i, -1, -1);
203 }
204
205 if (verbose > 2)
8cec1ead
WD		 206--- main.c 30 Jun 2004 07:27:30 -0000 1.202
207+++ main.c 30 Jun 2004 07:40:25 -0000
208@@ -429,7 +429,7 @@ static int do_recv(int f_in,int f_out,st
8dd60649
WD		 209 {
210 int pid;
8cec1ead 211 int status = 0;
8dd60649 212- int error_pipe[2];
8dd60649
WD		 213+ int error_pipe[2], name_pipe[2];
214
215 if (preserve_hard_links)
216 init_hard_links(flist);
8cec1ead 217@@ -441,8 +441,8 @@ static int do_recv(int f_in,int f_out,st
8dd60649
WD		 218 }
219 }
220
221- if (fd_pair(error_pipe) < 0) {
222- rprintf(FERROR,"error pipe failed in do_recv\n");
223+ if (fd_pair(error_pipe) < 0 || fd_pair(name_pipe) < 0) {
224+ rprintf(FERROR, "fd_pair() failed in do_recv\n");
225 exit_cleanup(RERR_SOCKETIO);
226 }
227
8cec1ead 228@@ -450,8 +450,10 @@ static int do_recv(int f_in,int f_out,st
8dd60649 229
8cec1ead 230 if ((pid = do_fork()) == 0) {
8dd60649
WD		 231 close(error_pipe[0]);
232+ close(name_pipe[1]);
233 if (f_in != f_out)
234 close(f_out);
235+ set_blocking(name_pipe[0]);
236
237 /* we can't let two processes write to the socket at one time */
238 io_multiplexing_close();
239@@ -459,7 +461,7 @@ static int do_recv(int f_in,int f_out,st
240 /* set place to send errors */
241 set_msg_fd_out(error_pipe[1]);
242
243- recv_files(f_in,flist,local_name);
244+ recv_files(f_in, flist, local_name, name_pipe[0]);
245 io_flush(FULL_FLUSH);
246 report(f_in);
247
248@@ -475,14 +477,16 @@ static int do_recv(int f_in,int f_out,st
249 am_generator = 1;
250
251 close(error_pipe[1]);
252+ close(name_pipe[0]);
253 if (f_in != f_out)
254 close(f_in);
255+ set_blocking(name_pipe[1]);
256
257 io_start_buffering_out(f_out);
258
259 set_msg_fd_in(error_pipe[0]);
260
261- generate_files(f_out, flist, local_name);
262+ generate_files(f_out, flist, local_name, name_pipe[1]);
263
264 get_redo_num(); /* Read final MSG_DONE and any prior messages. */
265 report(-1);
09add1ae 266--- options.c 20 Jun 2004 19:47:05 -0000 1.157
8cec1ead 267+++ options.c 30 Jun 2004 07:40:26 -0000
09add1ae
WD		 268@@ -117,7 +117,8 @@ unsigned int backup_dir_remainder;
269
270 char *backup_suffix = NULL;
271 char *tmpdir = NULL;
272-char *compare_dest = NULL;
273+char *compare_dest[MAX_COMP_DEST+1];
274+int num_comp_dest = 0;
275 char *config_file = NULL;
276 char *shell_cmd = NULL;
277 char *log_format = NULL;
278@@ -139,6 +140,7 @@ char *batch_prefix = NULL;
279
280 static int daemon_opt; /* sets am_daemon after option error-reporting */
281 static int modify_window_set;
282+static int saw_compare_dest = 0;
283
284 /** Local address to bind. As a character string because it's
285 * interpreted by the IPv6 layer: should be a numeric IP4 or IP6
286@@ -308,7 +310,7 @@ void usage(enum logcode F)
287 }
288
289 enum {OPT_VERSION = 1000, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
290- OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_LINK_DEST,
291+ OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_COMPARE_DEST, OPT_LINK_DEST,
292 OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
293 OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT,
294 OPT_REFUSED_BASE = 9000};
295@@ -366,8 +368,8 @@ static struct poptOption long_options[]
296 {"max-delete", 0, POPT_ARG_INT, &max_delete, 0, 0, 0 },
297 {"timeout", 0, POPT_ARG_INT, &io_timeout, OPT_TIMEOUT, 0, 0 },
298 {"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
299- {"compare-dest", 0, POPT_ARG_STRING, &compare_dest, 0, 0, 0 },
300- {"link-dest", 0, POPT_ARG_STRING, &compare_dest, OPT_LINK_DEST, 0, 0 },
301+ {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
302+ {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
303 /* TODO: Should this take an optional int giving the compression level? */
304 {"compress", 'z', POPT_ARG_NONE, &do_compression, 0, 0, 0 },
305 {"daemon", 0, POPT_ARG_NONE, &daemon_opt, 0, 0, 0 },
306@@ -585,8 +587,36 @@ int parse_arguments(int *argc, const cha
307 select_timeout = io_timeout;
308 break;
309
310+ case OPT_COMPARE_DEST:
311+#if HAVE_LINK
312+ if (num_comp_dest >= MAX_COMP_DEST-1) {
313+ rprintf(FERROR, "ERROR: %s\n", "too many --compare-dest args given");
314+ return 0;
315+ }
316+ arg = poptGetOptArg(pc);
317+ if (sanitize_paths)
318+ arg = alloc_sanitize_path(arg, curr_dir);
319+ compare_dest[num_comp_dest++] = (char *)arg;
320+ saw_compare_dest = 1;
321+ break;
322+#else
323+ snprintf(err_buf, sizeof err_buf,
324+ "hard links are not supported on this %s\n",
325+ am_server ? "server" : "client");
326+ rprintf(FERROR, "ERROR: %s", err_buf);
327+ return 0;
328+#endif
329+
330 case OPT_LINK_DEST:
331 #if HAVE_LINK
332+ if (num_comp_dest >= MAX_COMP_DEST-1) {
90bcf59b 333+ rprintf(FERROR, "ERROR: %s\n", "too many --link-dest args given");
09add1ae
WD		 334+ return 0;
335+ }
336+ arg = poptGetOptArg(pc);
337+ if (sanitize_paths)
338+ arg = alloc_sanitize_path(arg, curr_dir);
339+ compare_dest[num_comp_dest++] = (char *)arg;
340 link_dest = 1;
341 break;
342 #else
343@@ -661,6 +691,11 @@ int parse_arguments(int *argc, const cha
344 exit_cleanup(RERR_SYNTAX);
345 }
346
347+ if (saw_compare_dest && link_dest) {
348+ rprintf(FINFO,
349+ "WARNING: promoting --compare-dest options to --link-dest.\n");
350+ }
351+
352 if (archive_mode) {
353 if (!files_from)
354 recurse = 1;
355@@ -689,8 +724,6 @@ int parse_arguments(int *argc, const cha
356 (*argv)[i] = alloc_sanitize_path((*argv)[i], NULL);
357 if (tmpdir)
358 tmpdir = alloc_sanitize_path(tmpdir, curr_dir);
359- if (compare_dest)
360- compare_dest = alloc_sanitize_path(compare_dest, curr_dir);
361 if (backup_dir)
362 backup_dir = alloc_sanitize_path(backup_dir, curr_dir);
363 if (files_from)
364@@ -785,8 +818,8 @@ int parse_arguments(int *argc, const cha
365 **/
366 void server_options(char **args,int *argc)
367 {
368+ static char argstr[50+MAX_COMP_DEST*2];
369 int ac = *argc;
370- static char argstr[50];
371 char *arg;
372
373 int i, x;
374@@ -968,13 +1001,16 @@ void server_options(char **args,int *arg
375 args[ac++] = tmpdir;
376 }
377
378- if (compare_dest && am_sender) {
379+ if (compare_dest[0] && am_sender) {
380 /* the server only needs this option if it is not the sender,
381 * and it may be an older version that doesn't know this
382 * option, so don't send it if client is the sender.
383 */
384- args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
385- args[ac++] = compare_dest;
386+ int i;
387+ for (i = 0; i < num_comp_dest; i++) {
388+ args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
389+ args[ac++] = compare_dest[i];
390+ }
391 }
392
393 if (files_from && (!am_sender || remote_filesfrom_file)) {
8cec1ead
WD		 394--- receiver.c 30 Jun 2004 07:27:30 -0000 1.84
395+++ receiver.c 30 Jun 2004 07:40:26 -0000
8dd60649 396@@ -36,7 +36,6 @@ extern int preserve_perms;
09add1ae
WD		 397 extern int cvs_exclude;
398 extern int io_error;
399 extern char *tmpdir;
400-extern char *compare_dest;
09add1ae
WD		 401 extern int make_backups;
402 extern int do_progress;
403 extern char *backup_dir;
8dd60649
WD		 404@@ -293,14 +292,15 @@ static int receive_data(int f_in,struct
405 * main routine for receiver process.
406 *
407 * Receiver process runs on the same host as the generator process. */
408-int recv_files(int f_in,struct file_list *flist,char *local_name)
409+int recv_files(int f_in, struct file_list *flist, char *local_name,
410+ int f_name)
411 {
412 int fd1,fd2;
413 STRUCT_STAT st;
414 char *fname, fbuf[MAXPATHLEN];
415 char template[MAXPATHLEN];
416 char fnametmp[MAXPATHLEN];
417- char *fnamecmp;
418+ char *fnamecmp, *cp;
419 char fnamecmpbuf[MAXPATHLEN];
420 struct map_struct *mapbuf;
09add1ae 421 struct file_struct *file;
8dd60649
WD		 422@@ -364,19 +364,19 @@ int recv_files(int f_in,struct file_list
423 if (verbose > 2)
424 rprintf(FINFO,"recv_files(%s)\n",fname);
425
426- fnamecmp = fname;
427+ for (cp = fnamecmpbuf; ; cp++) {
428+ if (read(f_name, cp, 1) <= 0) {
429+ rsyserr(FERROR, errno, "fname-pipe read failed");
430+ exit_cleanup(RERR_PROTOCOL);
431+ }
432+ if (!*cp)
433+ break;
434+ }
435+ fnamecmp = *fnamecmpbuf ? fnamecmpbuf : fname;
09add1ae 436
09add1ae
WD		 437 /* open the file */
438 fd1 = do_open(fnamecmp, O_RDONLY, 0);
439
440- if (fd1 == -1 && compare_dest != NULL) {
8dd60649
WD		 441- /* try the file at compare_dest instead */
442- pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
09add1ae 443- compare_dest, fname);
8dd60649
WD		 444- fnamecmp = fnamecmpbuf;
445- fd1 = do_open(fnamecmp, O_RDONLY, 0);
446- }
447-
448 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
449 rsyserr(FERROR, errno, "fstat %s failed",
450 full_fname(fnamecmp));
451@@ -385,7 +385,7 @@ int recv_files(int f_in,struct file_list
452 continue;
09add1ae 453 }
8dd60649
WD		 454
455- if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) {
456+ if (fd1 != -1 && S_ISDIR(st.st_mode) && !*fnamecmpbuf) {
457 /* this special handling for directories
458 * wouldn't be necessary if robust_rename()
459 * and the underlying robust_unlink could cope
09add1ae 460--- rsync.h 16 May 2004 07:28:24 -0000 1.204
8cec1ead 461+++ rsync.h 30 Jun 2004 07:40:26 -0000
09add1ae
WD		 462@@ -98,6 +98,8 @@
463
464 #define MAX_ARGS 1000
465
466+#define MAX_COMP_DEST 20
467+
468 #define MPLEX_BASE 7
469
470 #define NO_EXCLUDES 0
Matt McCutchen's repository for the maintained patches to rsync, the fast and versatile remote file copier