compare-dest.diff

   1 This patch allows multiple --compare-dest or --link-dest options to be
   2 used, making the transfer of some files more optimal.  Note that the
   3 algorithm does NOT search for the best match -- it stops at the first
   4 match and uses that as the basis file for the transfer, so be sure to
   5 order your arguments appropriately (the args are searched in the order
   6 they are suppled).
   7
   8 Before compiling, be sure to run "make proto".
   9
  10 --- generator.c 29 Jun 2004 19:19:00 -0000      1.92
  11 +++ generator.c 30 Jun 2004 06:50:26 -0000
  12 @@ -42,7 +42,7 @@ extern int size_only;
  13  extern int io_timeout;
  14  extern int protocol_version;
  15  extern int always_checksum;
  16 -extern char *compare_dest;
  17 +extern char *compare_dest[];
  18  extern int link_dest;
  19  extern int whole_file;
  20  extern int local_server;
  21 @@ -80,13 +80,12 @@ static int skip_file(char *fname, struct
  22         if (always_checksum && S_ISREG(st->st_mode)) {
  23                 char sum[MD4_SUM_LENGTH];
  24                 char fnamecmpdest[MAXPATHLEN];
  25 +               int i;
  26
  27 -               if (compare_dest != NULL) {
  28 -                       if (access(fname, 0) != 0) {
  29 -                               pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  30 -                                        compare_dest, fname);
  31 -                               fname = fnamecmpdest;
  32 -                       }
  33 +               for (i = 0; compare_dest[i] != NULL && access(fname, 0) < 0; i++) {
  34 +                       pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  35 +                                       compare_dest[i], fname);
  36 +                       fname = fnamecmpdest;
  37                 }
  38                 file_checksum(fname,sum,st->st_size);
  39                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  40 @@ -267,7 +266,7 @@ static void generate_and_send_sums(struc
  41   * out.  It might be wrong.
  42   */
  43  static void recv_generator(char *fname, struct file_struct *file, int i,
  44 -                          int f_out)
  45 +                          int f_out, int f_nameout)
  46  {
  47         int fd;
  48         STRUCT_STAT st;
  49 @@ -424,15 +423,22 @@ static void recv_generator(char *fname,
  50
  51         fnamecmp = fname;
  52
  53 -       if (statret == -1 && compare_dest != NULL) {
  54 +       if (statret == -1 && compare_dest[0] != NULL) {
  55                 /* try the file at compare_dest instead */
  56                 int saveerrno = errno;
  57 -               pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
  58 -               statret = link_stat(fnamecmpbuf, &st, 0);
  59 -               if (!S_ISREG(st.st_mode))
  60 -                       statret = -1;
  61 -               if (statret == -1)
  62 +               int i;
  63 +               for (i = 0; compare_dest[i] != NULL; i++) {
  64 +                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest[i], fname);
  65 +                       if ((statret = link_stat(fnamecmpbuf, &st, 0)) == 0) {
  66 +                               if (S_ISREG(st.st_mode))
  67 +                                       break;
  68 +                               statret = -1;
  69 +                       }
  70 +               }
  71 +               if (statret < 0) {
  72                         errno = saveerrno;
  73 +                       *fnamecmpbuf = '\0';
  74 +               }
  75  #if HAVE_LINK
  76                 else if (link_dest && !dry_run) {
  77                         if (do_link(fnamecmpbuf, fname) != 0) {
  78 @@ -440,18 +446,22 @@ static void recv_generator(char *fname,
  79                                         rsyserr(FINFO, errno, "link %s => %s",
  80                                                 fnamecmpbuf, fname);
  81                                 }
  82 -                       }
  83 -                       fnamecmp = fnamecmpbuf;
  84 +                               fnamecmp = fnamecmpbuf;
  85 +                       } else
  86 +                               *fnamecmpbuf = '\0';
  87                 }
  88  #endif
  89                 else
  90                         fnamecmp = fnamecmpbuf;
  91 -       }
  92 +       } else
  93 +               *fnamecmpbuf = '\0';
  94
  95         if (statret == -1) {
  96                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  97                         return;
  98                 if (errno == ENOENT) {
  99 +                       if (f_nameout >= 0)
 100 +                               write(f_nameout, "", 1);
 101                         write_int(f_out,i);
 102                         if (!dry_run)
 103                                 write_sum_head(f_out, NULL);
 104 @@ -471,19 +481,21 @@ static void recv_generator(char *fname,
 105                 /* now pretend the file didn't exist */
 106                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 107                         return;
 108 +               if (f_nameout >= 0)
 109 +                       write(f_nameout, "", 1);
 110                 write_int(f_out,i);
 111                 if (!dry_run)
 112                         write_sum_head(f_out, NULL);
 113                 return;
 114         }
 115
 116 -       if (opt_ignore_existing && fnamecmp == fname) {
 117 +       if (opt_ignore_existing && !*fnamecmpbuf) {
 118                 if (verbose > 1)
 119                         rprintf(FINFO,"%s exists\n",fname);
 120                 return;
 121         }
 122
 123 -       if (update_only && fnamecmp == fname
 124 +       if (update_only && !*fnamecmpbuf
 125             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 126                 if (verbose > 1)
 127                         rprintf(FINFO,"%s is newer\n",fname);
 128 @@ -491,17 +503,21 @@ static void recv_generator(char *fname,
 129         }
 130
 131         if (skip_file(fname, file, &st)) {
 132 -               if (fnamecmp == fname)
 133 +               if (!*fnamecmpbuf)
 134                         set_perms(fname, file, &st, PERMS_REPORT);
 135                 return;
 136         }
 137
 138         if (dry_run) {
 139 +               if (f_nameout >= 0)
 140 +                       write(f_nameout, "", 1);
 141                 write_int(f_out,i);
 142                 return;
 143         }
 144
 145         if (disable_deltas_p()) {
 146 +               if (f_nameout >= 0)
 147 +                       write(f_nameout, "", 1);
 148                 write_int(f_out,i);
 149                 write_sum_head(f_out, NULL);
 150                 return;
 151 @@ -516,6 +532,8 @@ static void recv_generator(char *fname,
 152                 /* pretend the file didn't exist */
 153                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 154                         return;
 155 +               if (f_nameout >= 0)
 156 +                       write(f_nameout, "", 1);
 157                 write_int(f_out,i);
 158                 write_sum_head(f_out, NULL);
 159                 return;
 160 @@ -534,6 +552,8 @@ static void recv_generator(char *fname,
 161         if (verbose > 2)
 162                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 163
 164 +       if (f_nameout >= 0)
 165 +               write(f_nameout, fnamecmpbuf, strlen(fnamecmpbuf) + 1);
 166         write_int(f_out,i);
 167         generate_and_send_sums(mapbuf, st.st_size, f_out);
 168
 169 @@ -543,10 +563,11 @@ static void recv_generator(char *fname,
 170  }
 171
 172
 173 -void generate_files(int f, struct file_list *flist, char *local_name)
 174 +void generate_files(int f, struct file_list *flist, char *local_name,
 175 +                   int f_nameout)
 176  {
 177         int i;
 178 -       int phase=0;
 179 +       int phase = 0;
 180         char fbuf[MAXPATHLEN];
 181
 182         if (verbose > 2) {
 183 @@ -584,7 +605,7 @@ void generate_files(int f, struct file_l
 184                 }
 185
 186                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 187 -                              file, i, f);
 188 +                              file, i, f, f_nameout);
 189         }
 190
 191         phase++;
 192 @@ -601,7 +622,7 @@ void generate_files(int f, struct file_l
 193         while ((i = get_redo_num()) != -1) {
 194                 struct file_struct *file = flist->files[i];
 195                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 196 -                              file, i, f);
 197 +                              file, i, f, f_nameout);
 198         }
 199
 200         phase++;
 201 @@ -620,7 +641,7 @@ void generate_files(int f, struct file_l
 202                 if (!file->basename || !S_ISDIR(file->mode))
 203                         continue;
 204                 recv_generator(local_name ? local_name : f_name(file),
 205 -                              file, i, -1);
 206 +                              file, i, -1, -1);
 207         }
 208
 209         if (verbose > 2)
 210 --- main.c      28 Jun 2004 17:45:40 -0000      1.201
 211 +++ main.c      30 Jun 2004 06:50:27 -0000
 212 @@ -428,8 +428,8 @@ static void do_server_sender(int f_in, i
 213  static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name)
 214  {
 215         int pid;
 216 -       int status=0;
 217 -       int error_pipe[2];
 218 +       int status = 0;
 219 +       int error_pipe[2], name_pipe[2];
 220
 221         if (preserve_hard_links)
 222                 init_hard_links(flist);
 223 @@ -441,17 +441,19 @@ static int do_recv(int f_in,int f_out,st
 224                 }
 225         }
 226
 227 -       if (fd_pair(error_pipe) < 0) {
 228 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 229 +       if (fd_pair(error_pipe) < 0 || fd_pair(name_pipe) < 0) {
 230 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 231                 exit_cleanup(RERR_SOCKETIO);
 232         }
 233
 234         io_flush(NORMAL_FLUSH);
 235
 236 -       if ((pid=do_fork()) == 0) {
 237 +       if ((pid = do_fork()) == 0) {
 238                 close(error_pipe[0]);
 239 +               close(name_pipe[1]);
 240                 if (f_in != f_out)
 241                         close(f_out);
 242 +               set_blocking(name_pipe[0]);
 243
 244                 /* we can't let two processes write to the socket at one time */
 245                 io_multiplexing_close();
 246 @@ -459,7 +461,7 @@ static int do_recv(int f_in,int f_out,st
 247                 /* set place to send errors */
 248                 set_msg_fd_out(error_pipe[1]);
 249
 250 -               recv_files(f_in,flist,local_name);
 251 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 252                 io_flush(FULL_FLUSH);
 253                 report(f_in);
 254
 255 @@ -475,14 +477,16 @@ static int do_recv(int f_in,int f_out,st
 256         am_generator = 1;
 257
 258         close(error_pipe[1]);
 259 +       close(name_pipe[0]);
 260         if (f_in != f_out)
 261                 close(f_in);
 262 +       set_blocking(name_pipe[1]);
 263
 264         io_start_buffering_out(f_out);
 265
 266         set_msg_fd_in(error_pipe[0]);
 267
 268 -       generate_files(f_out, flist, local_name);
 269 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 270
 271         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 272         report(-1);
 273 --- options.c   20 Jun 2004 19:47:05 -0000      1.157
 274 +++ options.c   30 Jun 2004 06:50:27 -0000
 275 @@ -117,7 +117,8 @@ unsigned int backup_dir_remainder;
 276
 277  char *backup_suffix = NULL;
 278  char *tmpdir = NULL;
 279 -char *compare_dest = NULL;
 280 +char *compare_dest[MAX_COMP_DEST+1];
 281 +int num_comp_dest = 0;
 282  char *config_file = NULL;
 283  char *shell_cmd = NULL;
 284  char *log_format = NULL;
 285 @@ -139,6 +140,7 @@ char *batch_prefix = NULL;
 286
 287  static int daemon_opt;   /* sets am_daemon after option error-reporting */
 288  static int modify_window_set;
 289 +static int saw_compare_dest = 0;
 290
 291  /** Local address to bind.  As a character string because it's
 292   * interpreted by the IPv6 layer: should be a numeric IP4 or IP6
 293 @@ -308,7 +310,7 @@ void usage(enum logcode F)
 294  }
 295
 296  enum {OPT_VERSION = 1000, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
 297 -      OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_LINK_DEST,
 298 +      OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_COMPARE_DEST, OPT_LINK_DEST,
 299        OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
 300        OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT,
 301        OPT_REFUSED_BASE = 9000};
 302 @@ -366,8 +368,8 @@ static struct poptOption long_options[]
 303    {"max-delete",       0,  POPT_ARG_INT,    &max_delete, 0, 0, 0 },
 304    {"timeout",          0,  POPT_ARG_INT,    &io_timeout, OPT_TIMEOUT, 0, 0 },
 305    {"temp-dir",        'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
 306 -  {"compare-dest",     0,  POPT_ARG_STRING, &compare_dest, 0, 0, 0 },
 307 -  {"link-dest",        0,  POPT_ARG_STRING, &compare_dest,  OPT_LINK_DEST, 0, 0 },
 308 +  {"compare-dest",     0,  POPT_ARG_STRING, 0,              OPT_COMPARE_DEST, 0, 0 },
 309 +  {"link-dest",        0,  POPT_ARG_STRING, 0,              OPT_LINK_DEST, 0, 0 },
 310    /* TODO: Should this take an optional int giving the compression level? */
 311    {"compress",        'z', POPT_ARG_NONE,   &do_compression, 0, 0, 0 },
 312    {"daemon",           0,  POPT_ARG_NONE,   &daemon_opt, 0, 0, 0 },
 313 @@ -585,8 +587,36 @@ int parse_arguments(int *argc, const cha
 314                                 select_timeout = io_timeout;
 315                         break;
 316
 317 +               case OPT_COMPARE_DEST:
 318 +#if HAVE_LINK
 319 +                       if (num_comp_dest >= MAX_COMP_DEST-1) {
 320 +                               rprintf(FERROR, "ERROR: %s\n", "too many --compare-dest args given");
 321 +                               return 0;
 322 +                       }
 323 +                       arg = poptGetOptArg(pc);
 324 +                       if (sanitize_paths)
 325 +                               arg = alloc_sanitize_path(arg, curr_dir);
 326 +                       compare_dest[num_comp_dest++] = (char *)arg;
 327 +                       saw_compare_dest = 1;
 328 +                       break;
 329 +#else
 330 +                       snprintf(err_buf, sizeof err_buf,
 331 +                                "hard links are not supported on this %s\n",
 332 +                                am_server ? "server" : "client");
 333 +                       rprintf(FERROR, "ERROR: %s", err_buf);
 334 +                       return 0;
 335 +#endif
 336 +
 337                 case OPT_LINK_DEST:
 338  #if HAVE_LINK
 339 +                       if (num_comp_dest >= MAX_COMP_DEST-1) {
 340 +                               rprintf(FERROR, "ERROR: %s\n", "too many --link-dest args given");
 341 +                               return 0;
 342 +                       }
 343 +                       arg = poptGetOptArg(pc);
 344 +                       if (sanitize_paths)
 345 +                               arg = alloc_sanitize_path(arg, curr_dir);
 346 +                       compare_dest[num_comp_dest++] = (char *)arg;
 347                         link_dest = 1;
 348                         break;
 349  #else
 350 @@ -661,6 +691,11 @@ int parse_arguments(int *argc, const cha
 351                 exit_cleanup(RERR_SYNTAX);
 352         }
 353
 354 +       if (saw_compare_dest && link_dest) {
 355 +               rprintf(FINFO,
 356 +                       "WARNING: promoting --compare-dest options to --link-dest.\n");
 357 +       }
 358 +
 359         if (archive_mode) {
 360                 if (!files_from)
 361                         recurse = 1;
 362 @@ -689,8 +724,6 @@ int parse_arguments(int *argc, const cha
 363                         (*argv)[i] = alloc_sanitize_path((*argv)[i], NULL);
 364                 if (tmpdir)
 365                         tmpdir = alloc_sanitize_path(tmpdir, curr_dir);
 366 -               if (compare_dest)
 367 -                       compare_dest = alloc_sanitize_path(compare_dest, curr_dir);
 368                 if (backup_dir)
 369                         backup_dir = alloc_sanitize_path(backup_dir, curr_dir);
 370                 if (files_from)
 371 @@ -785,8 +818,8 @@ int parse_arguments(int *argc, const cha
 372   **/
 373  void server_options(char **args,int *argc)
 374  {
 375 +       static char argstr[50+MAX_COMP_DEST*2];
 376         int ac = *argc;
 377 -       static char argstr[50];
 378         char *arg;
 379
 380         int i, x;
 381 @@ -968,13 +1001,16 @@ void server_options(char **args,int *arg
 382                 args[ac++] = tmpdir;
 383         }
 384
 385 -       if (compare_dest && am_sender) {
 386 +       if (compare_dest[0] && am_sender) {
 387                 /* the server only needs this option if it is not the sender,
 388                  *   and it may be an older version that doesn't know this
 389                  *   option, so don't send it if client is the sender.
 390                  */
 391 -               args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
 392 -               args[ac++] = compare_dest;
 393 +               int i;
 394 +               for (i = 0; i < num_comp_dest; i++) {
 395 +                       args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
 396 +                       args[ac++] = compare_dest[i];
 397 +               }
 398         }
 399
 400         if (files_from && (!am_sender || remote_filesfrom_file)) {
 401 --- receiver.c  29 Jun 2004 15:12:01 -0000      1.83
 402 +++ receiver.c  30 Jun 2004 06:50:27 -0000
 403 @@ -36,7 +36,6 @@ extern int preserve_perms;
 404  extern int cvs_exclude;
 405  extern int io_error;
 406  extern char *tmpdir;
 407 -extern char *compare_dest;
 408  extern int make_backups;
 409  extern int do_progress;
 410  extern char *backup_dir;
 411 @@ -293,14 +292,15 @@ static int receive_data(int f_in,struct
 412   * main routine for receiver process.
 413   *
 414   * Receiver process runs on the same host as the generator process. */
 415 -int recv_files(int f_in,struct file_list *flist,char *local_name)
 416 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 417 +              int f_name)
 418  {
 419         int fd1,fd2;
 420         STRUCT_STAT st;
 421         char *fname, fbuf[MAXPATHLEN];
 422         char template[MAXPATHLEN];
 423         char fnametmp[MAXPATHLEN];
 424 -       char *fnamecmp;
 425 +       char *fnamecmp, *cp;
 426         char fnamecmpbuf[MAXPATHLEN];
 427         struct map_struct *mapbuf;
 428         struct file_struct *file;
 429 @@ -364,19 +364,19 @@ int recv_files(int f_in,struct file_list
 430                 if (verbose > 2)
 431                         rprintf(FINFO,"recv_files(%s)\n",fname);
 432
 433 -               fnamecmp = fname;
 434 +               for (cp = fnamecmpbuf; ; cp++) {
 435 +                       if (read(f_name, cp, 1) <= 0) {
 436 +                               rsyserr(FERROR, errno, "fname-pipe read failed");
 437 +                               exit_cleanup(RERR_PROTOCOL);
 438 +                       }
 439 +                       if (!*cp)
 440 +                               break;
 441 +               }
 442 +               fnamecmp = *fnamecmpbuf ? fnamecmpbuf : fname;
 443
 444                 /* open the file */
 445                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 446
 447 -               if (fd1 == -1 && compare_dest != NULL) {
 448 -                       /* try the file at compare_dest instead */
 449 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 450 -                                compare_dest, fname);
 451 -                       fnamecmp = fnamecmpbuf;
 452 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 453 -               }
 454 -
 455                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 456                         rsyserr(FERROR, errno, "fstat %s failed",
 457                                 full_fname(fnamecmp));
 458 @@ -385,7 +385,7 @@ int recv_files(int f_in,struct file_list
 459                         continue;
 460                 }
 461
 462 -               if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) {
 463 +               if (fd1 != -1 && S_ISDIR(st.st_mode) && !*fnamecmpbuf) {
 464                         /* this special handling for directories
 465                          * wouldn't be necessary if robust_rename()
 466                          * and the underlying robust_unlink could cope
 467 --- rsync.h     16 May 2004 07:28:24 -0000      1.204
 468 +++ rsync.h     30 Jun 2004 06:50:27 -0000
 469 @@ -98,6 +98,8 @@
 470
 471  #define MAX_ARGS 1000
 472
 473 +#define MAX_COMP_DEST 20
 474 +
 475  #define MPLEX_BASE 7
 476
 477  #define NO_EXCLUDES    0