compare-dest.diff

   1 This patch allows multiple --compare-dest or --link-dest options to be
   2 used, making the transfer of some files more optimal.  Note that the
   3 algorithm does NOT search for the best match -- it stops at the first
   4 match and uses that as the basis file for the transfer, so be sure to
   5 order your arguments appropriately (the args are searched in the order
   6 they are suppled).
   7
   8 Before compiling, be sure to run "make proto".
   9
  10 --- generator.c 30 Jun 2004 07:27:30 -0000      1.93
  11 +++ generator.c 30 Jun 2004 07:40:25 -0000
  12 @@ -42,7 +42,7 @@ extern int size_only;
  13  extern int io_timeout;
  14  extern int protocol_version;
  15  extern int always_checksum;
  16 -extern char *compare_dest;
  17 +extern char *compare_dest[];
  18  extern int link_dest;
  19  extern int whole_file;
  20  extern int local_server;
  21 @@ -80,13 +80,12 @@ static int skip_file(char *fname, struct
  22         if (always_checksum && S_ISREG(st->st_mode)) {
  23                 char sum[MD4_SUM_LENGTH];
  24                 char fnamecmpdest[MAXPATHLEN];
  25 +               int i;
  26
  27 -               if (compare_dest != NULL) {
  28 -                       if (access(fname, 0) != 0) {
  29 -                               pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  30 -                                        compare_dest, fname);
  31 -                               fname = fnamecmpdest;
  32 -                       }
  33 +               for (i = 0; compare_dest[i] != NULL && access(fname, 0) < 0; i++) {
  34 +                       pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  35 +                                       compare_dest[i], fname);
  36 +                       fname = fnamecmpdest;
  37                 }
  38                 file_checksum(fname,sum,st->st_size);
  39                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  40 @@ -267,7 +266,7 @@ static void generate_and_send_sums(struc
  41   * out.  It might be wrong.
  42   */
  43  static void recv_generator(char *fname, struct file_struct *file, int i,
  44 -                          int f_out)
  45 +                          int f_out, int f_nameout)
  46  {
  47         int fd;
  48         STRUCT_STAT st;
  49 @@ -424,15 +423,22 @@ static void recv_generator(char *fname,
  50
  51         fnamecmp = fname;
  52
  53 -       if (statret == -1 && compare_dest != NULL) {
  54 +       if (statret == -1 && compare_dest[0] != NULL) {
  55                 /* try the file at compare_dest instead */
  56                 int saveerrno = errno;
  57 -               pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
  58 -               statret = link_stat(fnamecmpbuf, &st, 0);
  59 -               if (!S_ISREG(st.st_mode))
  60 -                       statret = -1;
  61 -               if (statret == -1)
  62 +               int i;
  63 +               for (i = 0; compare_dest[i] != NULL; i++) {
  64 +                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest[i], fname);
  65 +                       if ((statret = link_stat(fnamecmpbuf, &st, 0)) == 0) {
  66 +                               if (S_ISREG(st.st_mode))
  67 +                                       break;
  68 +                               statret = -1;
  69 +                       }
  70 +               }
  71 +               if (statret < 0) {
  72                         errno = saveerrno;
  73 +                       *fnamecmpbuf = '\0';
  74 +               }
  75  #if HAVE_LINK
  76                 else if (link_dest && !dry_run) {
  77                         if (do_link(fnamecmpbuf, fname) != 0) {
  78 @@ -440,18 +446,22 @@ static void recv_generator(char *fname,
  79                                         rsyserr(FINFO, errno, "link %s => %s",
  80                                                 fnamecmpbuf, fname);
  81                                 }
  82 -                       }
  83 -                       fnamecmp = fnamecmpbuf;
  84 +                               fnamecmp = fnamecmpbuf;
  85 +                       } else
  86 +                               *fnamecmpbuf = '\0';
  87                 }
  88  #endif
  89                 else
  90                         fnamecmp = fnamecmpbuf;
  91 -       }
  92 +       } else
  93 +               *fnamecmpbuf = '\0';
  94
  95         if (statret == -1) {
  96                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  97                         return;
  98                 if (errno == ENOENT) {
  99 +                       if (f_nameout >= 0)
 100 +                               write(f_nameout, "", 1);
 101                         write_int(f_out,i);
 102                         if (!dry_run)
 103                                 write_sum_head(f_out, NULL);
 104 @@ -471,19 +481,21 @@ static void recv_generator(char *fname,
 105                 /* now pretend the file didn't exist */
 106                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 107                         return;
 108 +               if (f_nameout >= 0)
 109 +                       write(f_nameout, "", 1);
 110                 write_int(f_out,i);
 111                 if (!dry_run)
 112                         write_sum_head(f_out, NULL);
 113                 return;
 114         }
 115
 116 -       if (opt_ignore_existing && fnamecmp == fname) {
 117 +       if (opt_ignore_existing && !*fnamecmpbuf) {
 118                 if (verbose > 1)
 119                         rprintf(FINFO,"%s exists\n",fname);
 120                 return;
 121         }
 122
 123 -       if (update_only && fnamecmp == fname
 124 +       if (update_only && !*fnamecmpbuf
 125             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 126                 if (verbose > 1)
 127                         rprintf(FINFO,"%s is newer\n",fname);
 128 @@ -491,17 +503,21 @@ static void recv_generator(char *fname,
 129         }
 130
 131         if (skip_file(fname, file, &st)) {
 132 -               if (fnamecmp == fname)
 133 +               if (!*fnamecmpbuf)
 134                         set_perms(fname, file, &st, PERMS_REPORT);
 135                 return;
 136         }
 137
 138         if (dry_run) {
 139 +               if (f_nameout >= 0)
 140 +                       write(f_nameout, "", 1);
 141                 write_int(f_out,i);
 142                 return;
 143         }
 144
 145         if (disable_deltas_p()) {
 146 +               if (f_nameout >= 0)
 147 +                       write(f_nameout, "", 1);
 148                 write_int(f_out,i);
 149                 write_sum_head(f_out, NULL);
 150                 return;
 151 @@ -516,6 +532,8 @@ static void recv_generator(char *fname,
 152                 /* pretend the file didn't exist */
 153                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 154                         return;
 155 +               if (f_nameout >= 0)
 156 +                       write(f_nameout, "", 1);
 157                 write_int(f_out,i);
 158                 write_sum_head(f_out, NULL);
 159                 return;
 160 @@ -534,6 +552,8 @@ static void recv_generator(char *fname,
 161         if (verbose > 2)
 162                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 163
 164 +       if (f_nameout >= 0)
 165 +               write(f_nameout, fnamecmpbuf, strlen(fnamecmpbuf) + 1);
 166         write_int(f_out,i);
 167         generate_and_send_sums(mapbuf, st.st_size, f_out);
 168
 169 @@ -543,7 +563,8 @@ static void recv_generator(char *fname,
 170  }
 171
 172
 173 -void generate_files(int f, struct file_list *flist, char *local_name)
 174 +void generate_files(int f, struct file_list *flist, char *local_name,
 175 +                   int f_nameout)
 176  {
 177         int i;
 178         int phase = 0;
 179 @@ -584,7 +605,7 @@ void generate_files(int f, struct file_l
 180                 }
 181
 182                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 183 -                              file, i, f);
 184 +                              file, i, f, f_nameout);
 185         }
 186
 187         phase++;
 188 @@ -601,7 +622,7 @@ void generate_files(int f, struct file_l
 189         while ((i = get_redo_num()) != -1) {
 190                 struct file_struct *file = flist->files[i];
 191                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 192 -                              file, i, f);
 193 +                              file, i, f, f_nameout);
 194         }
 195
 196         phase++;
 197 @@ -620,7 +641,7 @@ void generate_files(int f, struct file_l
 198                 if (!file->basename || !S_ISDIR(file->mode))
 199                         continue;
 200                 recv_generator(local_name ? local_name : f_name(file),
 201 -                              file, i, -1);
 202 +                              file, i, -1, -1);
 203         }
 204
 205         if (verbose > 2)
 206 --- main.c      30 Jun 2004 07:27:30 -0000      1.202
 207 +++ main.c      30 Jun 2004 07:40:25 -0000
 208 @@ -429,7 +429,7 @@ static int do_recv(int f_in,int f_out,st
 209  {
 210         int pid;
 211         int status = 0;
 212 -       int error_pipe[2];
 213 +       int error_pipe[2], name_pipe[2];
 214
 215         if (preserve_hard_links)
 216                 init_hard_links(flist);
 217 @@ -441,8 +441,8 @@ static int do_recv(int f_in,int f_out,st
 218                 }
 219         }
 220
 221 -       if (fd_pair(error_pipe) < 0) {
 222 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 223 +       if (fd_pair(error_pipe) < 0 || fd_pair(name_pipe) < 0) {
 224 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 225                 exit_cleanup(RERR_SOCKETIO);
 226         }
 227
 228 @@ -450,8 +450,10 @@ static int do_recv(int f_in,int f_out,st
 229
 230         if ((pid = do_fork()) == 0) {
 231                 close(error_pipe[0]);
 232 +               close(name_pipe[1]);
 233                 if (f_in != f_out)
 234                         close(f_out);
 235 +               set_blocking(name_pipe[0]);
 236
 237                 /* we can't let two processes write to the socket at one time */
 238                 io_multiplexing_close();
 239 @@ -459,7 +461,7 @@ static int do_recv(int f_in,int f_out,st
 240                 /* set place to send errors */
 241                 set_msg_fd_out(error_pipe[1]);
 242
 243 -               recv_files(f_in,flist,local_name);
 244 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 245                 io_flush(FULL_FLUSH);
 246                 report(f_in);
 247
 248 @@ -475,14 +477,16 @@ static int do_recv(int f_in,int f_out,st
 249         am_generator = 1;
 250
 251         close(error_pipe[1]);
 252 +       close(name_pipe[0]);
 253         if (f_in != f_out)
 254                 close(f_in);
 255 +       set_blocking(name_pipe[1]);
 256
 257         io_start_buffering_out(f_out);
 258
 259         set_msg_fd_in(error_pipe[0]);
 260
 261 -       generate_files(f_out, flist, local_name);
 262 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 263
 264         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 265         report(-1);
 266 --- options.c   20 Jun 2004 19:47:05 -0000      1.157
 267 +++ options.c   30 Jun 2004 07:40:26 -0000
 268 @@ -117,7 +117,8 @@ unsigned int backup_dir_remainder;
 269
 270  char *backup_suffix = NULL;
 271  char *tmpdir = NULL;
 272 -char *compare_dest = NULL;
 273 +char *compare_dest[MAX_COMP_DEST+1];
 274 +int num_comp_dest = 0;
 275  char *config_file = NULL;
 276  char *shell_cmd = NULL;
 277  char *log_format = NULL;
 278 @@ -139,6 +140,7 @@ char *batch_prefix = NULL;
 279
 280  static int daemon_opt;   /* sets am_daemon after option error-reporting */
 281  static int modify_window_set;
 282 +static int saw_compare_dest = 0;
 283
 284  /** Local address to bind.  As a character string because it's
 285   * interpreted by the IPv6 layer: should be a numeric IP4 or IP6
 286 @@ -308,7 +310,7 @@ void usage(enum logcode F)
 287  }
 288
 289  enum {OPT_VERSION = 1000, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
 290 -      OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_LINK_DEST,
 291 +      OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_COMPARE_DEST, OPT_LINK_DEST,
 292        OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
 293        OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT,
 294        OPT_REFUSED_BASE = 9000};
 295 @@ -366,8 +368,8 @@ static struct poptOption long_options[]
 296    {"max-delete",       0,  POPT_ARG_INT,    &max_delete, 0, 0, 0 },
 297    {"timeout",          0,  POPT_ARG_INT,    &io_timeout, OPT_TIMEOUT, 0, 0 },
 298    {"temp-dir",        'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
 299 -  {"compare-dest",     0,  POPT_ARG_STRING, &compare_dest, 0, 0, 0 },
 300 -  {"link-dest",        0,  POPT_ARG_STRING, &compare_dest,  OPT_LINK_DEST, 0, 0 },
 301 +  {"compare-dest",     0,  POPT_ARG_STRING, 0,              OPT_COMPARE_DEST, 0, 0 },
 302 +  {"link-dest",        0,  POPT_ARG_STRING, 0,              OPT_LINK_DEST, 0, 0 },
 303    /* TODO: Should this take an optional int giving the compression level? */
 304    {"compress",        'z', POPT_ARG_NONE,   &do_compression, 0, 0, 0 },
 305    {"daemon",           0,  POPT_ARG_NONE,   &daemon_opt, 0, 0, 0 },
 306 @@ -585,8 +587,36 @@ int parse_arguments(int *argc, const cha
 307                                 select_timeout = io_timeout;
 308                         break;
 309
 310 +               case OPT_COMPARE_DEST:
 311 +#if HAVE_LINK
 312 +                       if (num_comp_dest >= MAX_COMP_DEST-1) {
 313 +                               rprintf(FERROR, "ERROR: %s\n", "too many --compare-dest args given");
 314 +                               return 0;
 315 +                       }
 316 +                       arg = poptGetOptArg(pc);
 317 +                       if (sanitize_paths)
 318 +                               arg = alloc_sanitize_path(arg, curr_dir);
 319 +                       compare_dest[num_comp_dest++] = (char *)arg;
 320 +                       saw_compare_dest = 1;
 321 +                       break;
 322 +#else
 323 +                       snprintf(err_buf, sizeof err_buf,
 324 +                                "hard links are not supported on this %s\n",
 325 +                                am_server ? "server" : "client");
 326 +                       rprintf(FERROR, "ERROR: %s", err_buf);
 327 +                       return 0;
 328 +#endif
 329 +
 330                 case OPT_LINK_DEST:
 331  #if HAVE_LINK
 332 +                       if (num_comp_dest >= MAX_COMP_DEST-1) {
 333 +                               rprintf(FERROR, "ERROR: %s\n", "too many --link-dest args given");
 334 +                               return 0;
 335 +                       }
 336 +                       arg = poptGetOptArg(pc);
 337 +                       if (sanitize_paths)
 338 +                               arg = alloc_sanitize_path(arg, curr_dir);
 339 +                       compare_dest[num_comp_dest++] = (char *)arg;
 340                         link_dest = 1;
 341                         break;
 342  #else
 343 @@ -661,6 +691,11 @@ int parse_arguments(int *argc, const cha
 344                 exit_cleanup(RERR_SYNTAX);
 345         }
 346
 347 +       if (saw_compare_dest && link_dest) {
 348 +               rprintf(FINFO,
 349 +                       "WARNING: promoting --compare-dest options to --link-dest.\n");
 350 +       }
 351 +
 352         if (archive_mode) {
 353                 if (!files_from)
 354                         recurse = 1;
 355 @@ -689,8 +724,6 @@ int parse_arguments(int *argc, const cha
 356                         (*argv)[i] = alloc_sanitize_path((*argv)[i], NULL);
 357                 if (tmpdir)
 358                         tmpdir = alloc_sanitize_path(tmpdir, curr_dir);
 359 -               if (compare_dest)
 360 -                       compare_dest = alloc_sanitize_path(compare_dest, curr_dir);
 361                 if (backup_dir)
 362                         backup_dir = alloc_sanitize_path(backup_dir, curr_dir);
 363                 if (files_from)
 364 @@ -785,8 +818,8 @@ int parse_arguments(int *argc, const cha
 365   **/
 366  void server_options(char **args,int *argc)
 367  {
 368 +       static char argstr[50+MAX_COMP_DEST*2];
 369         int ac = *argc;
 370 -       static char argstr[50];
 371         char *arg;
 372
 373         int i, x;
 374 @@ -968,13 +1001,16 @@ void server_options(char **args,int *arg
 375                 args[ac++] = tmpdir;
 376         }
 377
 378 -       if (compare_dest && am_sender) {
 379 +       if (compare_dest[0] && am_sender) {
 380                 /* the server only needs this option if it is not the sender,
 381                  *   and it may be an older version that doesn't know this
 382                  *   option, so don't send it if client is the sender.
 383                  */
 384 -               args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
 385 -               args[ac++] = compare_dest;
 386 +               int i;
 387 +               for (i = 0; i < num_comp_dest; i++) {
 388 +                       args[ac++] = link_dest ? "--link-dest" : "--compare-dest";
 389 +                       args[ac++] = compare_dest[i];
 390 +               }
 391         }
 392
 393         if (files_from && (!am_sender || remote_filesfrom_file)) {
 394 --- receiver.c  30 Jun 2004 07:27:30 -0000      1.84
 395 +++ receiver.c  30 Jun 2004 07:40:26 -0000
 396 @@ -36,7 +36,6 @@ extern int preserve_perms;
 397  extern int cvs_exclude;
 398  extern int io_error;
 399  extern char *tmpdir;
 400 -extern char *compare_dest;
 401  extern int make_backups;
 402  extern int do_progress;
 403  extern char *backup_dir;
 404 @@ -293,14 +292,15 @@ static int receive_data(int f_in,struct
 405   * main routine for receiver process.
 406   *
 407   * Receiver process runs on the same host as the generator process. */
 408 -int recv_files(int f_in,struct file_list *flist,char *local_name)
 409 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 410 +              int f_name)
 411  {
 412         int fd1,fd2;
 413         STRUCT_STAT st;
 414         char *fname, fbuf[MAXPATHLEN];
 415         char template[MAXPATHLEN];
 416         char fnametmp[MAXPATHLEN];
 417 -       char *fnamecmp;
 418 +       char *fnamecmp, *cp;
 419         char fnamecmpbuf[MAXPATHLEN];
 420         struct map_struct *mapbuf;
 421         struct file_struct *file;
 422 @@ -364,19 +364,19 @@ int recv_files(int f_in,struct file_list
 423                 if (verbose > 2)
 424                         rprintf(FINFO,"recv_files(%s)\n",fname);
 425
 426 -               fnamecmp = fname;
 427 +               for (cp = fnamecmpbuf; ; cp++) {
 428 +                       if (read(f_name, cp, 1) <= 0) {
 429 +                               rsyserr(FERROR, errno, "fname-pipe read failed");
 430 +                               exit_cleanup(RERR_PROTOCOL);
 431 +                       }
 432 +                       if (!*cp)
 433 +                               break;
 434 +               }
 435 +               fnamecmp = *fnamecmpbuf ? fnamecmpbuf : fname;
 436
 437                 /* open the file */
 438                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 439
 440 -               if (fd1 == -1 && compare_dest != NULL) {
 441 -                       /* try the file at compare_dest instead */
 442 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 443 -                                compare_dest, fname);
 444 -                       fnamecmp = fnamecmpbuf;
 445 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 446 -               }
 447 -
 448                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 449                         rsyserr(FERROR, errno, "fstat %s failed",
 450                                 full_fname(fnamecmp));
 451 @@ -385,7 +385,7 @@ int recv_files(int f_in,struct file_list
 452                         continue;
 453                 }
 454
 455 -               if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) {
 456 +               if (fd1 != -1 && S_ISDIR(st.st_mode) && !*fnamecmpbuf) {
 457                         /* this special handling for directories
 458                          * wouldn't be necessary if robust_rename()
 459                          * and the underlying robust_unlink could cope
 460 --- rsync.h     16 May 2004 07:28:24 -0000      1.204
 461 +++ rsync.h     30 Jun 2004 07:40:26 -0000
 462 @@ -98,6 +98,8 @@
 463
 464  #define MAX_ARGS 1000
 465
 466 +#define MAX_COMP_DEST 20
 467 +
 468  #define MPLEX_BASE 7
 469
 470  #define NO_EXCLUDES    0