g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 You must run "make proto" before compiling.
   9
  10 --- orig/generator.c    2004-10-06 00:12:16
  11 +++ generator.c 2004-11-03 20:12:24
  12 @@ -254,14 +254,15 @@ static void generate_and_send_sums(int f
  13   * out.  It might be wrong.
  14   */
  15  static void recv_generator(char *fname, struct file_struct *file, int i,
  16 -                          int f_out)
  17 +                          int f_out, int f_out_name)
  18  {
  19 -       int fd, f_copy;
  20 +       int fd = -1, f_copy = -1;
  21         STRUCT_STAT st, partial_st;
  22 -       struct file_struct *back_file;
  23 +       struct file_struct *back_file = NULL;
  24         int statret, stat_errno;
  25 -       char *fnamecmp, *partialptr, *backupptr;
  26 +       char *fnamecmp, *partialptr, *backupptr = NULL;
  27         char fnamecmpbuf[MAXPATHLEN];
  28 +       uchar fnamecmp_type;
  29
  30         if (list_only)
  31                 return;
  32 @@ -421,6 +422,7 @@ static void recv_generator(char *fname,
  33         }
  34
  35         fnamecmp = fname;
  36 +       fnamecmp_type = FNAMECMP_FNAME;
  37
  38         if (statret == -1 && compare_dest != NULL) {
  39                 /* try the file at compare_dest instead */
  40 @@ -437,10 +439,14 @@ static void recv_generator(char *fname,
  41                                                         safe_fname(fname));
  42                                         }
  43                                         fnamecmp = fnamecmpbuf;
  44 +                                       fnamecmp_type = FNAMECMP_CMPDEST;
  45                                 }
  46                         } else
  47  #endif
  48 +                       {
  49                                 fnamecmp = fnamecmpbuf;
  50 +                               fnamecmp_type = FNAMECMP_CMPDEST;
  51 +                       }
  52                         statret = 0;
  53                 }
  54         }
  55 @@ -463,11 +469,9 @@ static void recv_generator(char *fname,
  56         if (statret == -1) {
  57                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  58                         return;
  59 -               if (stat_errno == ENOENT) {
  60 -                       write_int(f_out,i);
  61 -                       if (!dry_run && !read_batch)
  62 -                               write_sum_head(f_out, NULL);
  63 -               } else if (verbose > 1) {
  64 +               if (stat_errno == ENOENT)
  65 +                       goto notify_others;
  66 +               if (verbose > 1) {
  67                         rsyserr(FERROR, stat_errno,
  68                                 "recv_generator: failed to stat %s",
  69                                 full_fname(fname));
  70 @@ -475,13 +479,13 @@ static void recv_generator(char *fname,
  71                 return;
  72         }
  73
  74 -       if (opt_ignore_existing && fnamecmp == fname) {
  75 +       if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
  76                 if (verbose > 1)
  77                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
  78                 return;
  79         }
  80
  81 -       if (update_only && fnamecmp == fname
  82 +       if (update_only && fnamecmp_type == FNAMECMP_FNAME
  83             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
  84                 if (verbose > 1)
  85                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
  86 @@ -489,26 +493,23 @@ static void recv_generator(char *fname,
  87         }
  88
  89         if (skip_file(fnamecmp, file, &st)) {
  90 -               if (fnamecmp == fname)
  91 +               if (fnamecmp_type == FNAMECMP_FNAME)
  92                         set_perms(fname, file, &st, PERMS_REPORT);
  93                 return;
  94         }
  95
  96  prepare_to_open:
  97 -       if (dry_run || read_batch) {
  98 -               write_int(f_out,i);
  99 -               return;
 100 -       }
 101 -
 102 -       if (whole_file > 0) {
 103 -               write_int(f_out,i);
 104 -               write_sum_head(f_out, NULL);
 105 -               return;
 106 +       if (dry_run || whole_file > 0) {
 107 +               statret = -1;
 108 +               goto notify_others;
 109         }
 110 +       if (read_batch)
 111 +               goto notify_others;
 112
 113         if (partialptr) {
 114                 st = partial_st;
 115                 fnamecmp = partialptr;
 116 +               fnamecmp_type = FNAMECMP_PARTIAL_DIR;
 117         }
 118
 119         /* open the file */
 120 @@ -521,9 +522,8 @@ prepare_to_open:
 121                 /* pretend the file didn't exist */
 122                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 123                         return;
 124 -               write_int(f_out,i);
 125 -               write_sum_head(f_out, NULL);
 126 -               return;
 127 +               statret = -1;
 128 +               goto notify_others;
 129         }
 130
 131         if (inplace && make_backups) {
 132 @@ -550,10 +550,7 @@ prepare_to_open:
 133                         close(fd);
 134                         return;
 135                 }
 136 -       } else {
 137 -               backupptr = NULL;
 138 -               back_file = NULL;
 139 -               f_copy = -1;
 140 +               fnamecmp_type = FNAMECMP_BACKUP;
 141         }
 142
 143         if (verbose > 3) {
 144 @@ -564,22 +561,35 @@ prepare_to_open:
 145         if (verbose > 2)
 146                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 147
 148 -       write_int(f_out,i);
 149 -       generate_and_send_sums(fd, st.st_size, f_out, f_copy);
 150 -
 151 -       if (f_copy >= 0) {
 152 -               close(f_copy);
 153 -               set_perms(backupptr, back_file, NULL, 0);
 154 -               if (verbose > 1)
 155 -                       rprintf(FINFO, "backed up %s to %s\n", fname, backupptr);
 156 -               free(back_file);
 157 -       }
 158 +notify_others:
 159 +       write_int(f_out, i);
 160 +       if (f_out_name >= 0)
 161 +               write_byte(f_out_name, fnamecmp_type);
 162 +
 163 +       if (dry_run || read_batch)
 164 +               return;
 165 +
 166 +       if (statret == 0) {
 167 +               generate_and_send_sums(fd, st.st_size, f_out, f_copy);
 168 +
 169 +               if (f_copy >= 0) {
 170 +                       close(f_copy);
 171 +                       set_perms(backupptr, back_file, NULL, 0);
 172 +                       if (verbose > 1) {
 173 +                               rprintf(FINFO, "backed up %s to %s\n",
 174 +                                       fname, backupptr);
 175 +                       }
 176 +                       free(back_file);
 177 +               }
 178
 179 -       close(fd);
 180 +               close(fd);
 181 +       } else
 182 +               write_sum_head(f_out, NULL);
 183  }
 184
 185
 186 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 187 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 188 +                   int f_out_name)
 189  {
 190         int i;
 191         int phase = 0;
 192 @@ -620,7 +630,7 @@ void generate_files(int f_out, struct fi
 193                 }
 194
 195                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 196 -                              file, i, f_out);
 197 +                              file, i, f_out, f_out_name);
 198         }
 199
 200         phase++;
 201 @@ -637,7 +647,7 @@ void generate_files(int f_out, struct fi
 202         while ((i = get_redo_num()) != -1) {
 203                 struct file_struct *file = flist->files[i];
 204                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 205 -                              file, i, f_out);
 206 +                              file, i, f_out, f_out_name);
 207         }
 208
 209         phase++;
 210 @@ -656,7 +666,7 @@ void generate_files(int f_out, struct fi
 211                 if (!file->basename || !S_ISDIR(file->mode))
 212                         continue;
 213                 recv_generator(local_name ? local_name : f_name(file),
 214 -                              file, i, -1);
 215 +                              file, i, -1, -1);
 216         }
 217
 218         if (verbose > 2)
 219 --- orig/io.c   2004-08-02 02:44:26
 220 +++ io.c        2004-11-03 19:49:36
 221 @@ -857,6 +857,11 @@ static void writefd_unbuffered(int fd,ch
 222                         if (msg_fd_in > maxfd)
 223                                 maxfd = msg_fd_in;
 224                 }
 225 +               if (fd != sock_f_out && iobuf_out_cnt && no_flush == 1) {
 226 +                       FD_SET(sock_f_out, &w_fds);
 227 +                       if (sock_f_out > maxfd)
 228 +                               maxfd = sock_f_out;
 229 +               }
 230
 231                 tv.tv_sec = select_timeout;
 232                 tv.tv_usec = 0;
 233 @@ -875,8 +880,14 @@ static void writefd_unbuffered(int fd,ch
 234                 if (msg_fd_in >= 0 && FD_ISSET(msg_fd_in, &r_fds))
 235                         read_msg_fd();
 236
 237 -               if (!FD_ISSET(fd, &w_fds))
 238 +               if (!FD_ISSET(fd, &w_fds)) {
 239 +                       if (fd != sock_f_out && iobuf_out_cnt) {
 240 +                               no_flush--;
 241 +                               io_flush(NORMAL_FLUSH);
 242 +                               no_flush++;
 243 +                       }
 244                         continue;
 245 +               }
 246
 247                 n = len - total;
 248                 if (bwlimit && n > bwlimit_writemax)
 249 --- orig/main.c 2004-10-08 20:16:26
 250 +++ main.c      2004-11-03 16:07:53
 251 @@ -59,6 +59,7 @@ extern int filesfrom_fd;
 252  extern pid_t cleanup_child_pid;
 253  extern char *files_from;
 254  extern char *remote_filesfrom_file;
 255 +extern char *compare_dest;
 256  extern char *rsync_path;
 257  extern char *shell_cmd;
 258  extern char *batch_name;
 259 @@ -461,7 +462,8 @@ static int do_recv(int f_in,int f_out,st
 260  {
 261         int pid;
 262         int status = 0;
 263 -       int error_pipe[2];
 264 +       int error_pipe[2], name_pipe[2];
 265 +       BOOL need_name_pipe = compare_dest && !dry_run;
 266
 267         /* The receiving side mustn't obey this, or an existing symlink that
 268          * points to an identical file won't be replaced by the referent. */
 269 @@ -476,7 +478,8 @@ static int do_recv(int f_in,int f_out,st
 270                         delete_files(flist);
 271         }
 272
 273 -       if (fd_pair(error_pipe) < 0) {
 274 +       if (fd_pair(error_pipe) < 0
 275 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 276                 rsyserr(FERROR, errno, "pipe failed in do_recv");
 277                 exit_cleanup(RERR_SOCKETIO);
 278         }
 279 @@ -485,6 +488,11 @@ static int do_recv(int f_in,int f_out,st
 280
 281         if ((pid = do_fork()) == 0) {
 282                 close(error_pipe[0]);
 283 +               if (need_name_pipe) {
 284 +                       close(name_pipe[1]);
 285 +                       set_blocking(name_pipe[0]);
 286 +               } else
 287 +                       name_pipe[0] = -1;
 288                 if (f_in != f_out)
 289                         close(f_out);
 290
 291 @@ -494,7 +502,7 @@ static int do_recv(int f_in,int f_out,st
 292                 /* set place to send errors */
 293                 set_msg_fd_out(error_pipe[1]);
 294
 295 -               recv_files(f_in,flist,local_name);
 296 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 297                 io_flush(FULL_FLUSH);
 298                 report(f_in);
 299
 300 @@ -513,6 +521,11 @@ static int do_recv(int f_in,int f_out,st
 301                 stop_write_batch();
 302
 303         close(error_pipe[1]);
 304 +       if (need_name_pipe) {
 305 +               close(name_pipe[0]);
 306 +               set_nonblocking(name_pipe[1]);
 307 +       } else
 308 +               name_pipe[1] = -1;
 309         if (f_in != f_out)
 310                 close(f_in);
 311
 312 @@ -520,7 +533,7 @@ static int do_recv(int f_in,int f_out,st
 313
 314         set_msg_fd_in(error_pipe[0]);
 315
 316 -       generate_files(f_out, flist, local_name);
 317 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 318
 319         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 320         report(-1);
 321 --- orig/receiver.c     2004-11-03 11:56:04
 322 +++ receiver.c  2004-11-03 20:16:38
 323 @@ -30,6 +30,7 @@ extern int dry_run;
 324  extern int read_batch;
 325  extern int batch_gen_fd;
 326  extern int am_server;
 327 +extern int protocol_version;
 328  extern int relative_paths;
 329  extern int keep_dirlinks;
 330  extern int preserve_hard_links;
 331 @@ -329,7 +330,8 @@ static void discard_receive_data(int f_i
 332   * main routine for receiver process.
 333   *
 334   * Receiver process runs on the same host as the generator process. */
 335 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 336 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 337 +              int f_in_name)
 338  {
 339         int next_gen_i = -1;
 340         int fd1,fd2;
 341 @@ -358,8 +360,13 @@ int recv_files(int f_in, struct file_lis
 342                 i = read_int(f_in);
 343                 if (i == -1) {
 344                         if (read_batch) {
 345 -                               if (next_gen_i != flist->count)
 346 -                                       while (read_int(batch_gen_fd) != -1) {}
 347 +                               if (next_gen_i != flist->count) {
 348 +                                       do {
 349 +                                               if (f_in_name >= 0
 350 +                                                   && next_gen_i >= 0)
 351 +                                                       read_byte(f_in_name);
 352 +                                       } while (read_int(batch_gen_fd) != -1);
 353 +                               }
 354                                 next_gen_i = -1;
 355                         }
 356
 357 @@ -407,6 +414,8 @@ int recv_files(int f_in, struct file_lis
 358
 359                 if (read_batch) {
 360                         while (i > next_gen_i) {
 361 +                               if (f_in_name >= 0 && next_gen_i >= 0)
 362 +                                       read_byte(f_in_name);
 363                                 next_gen_i = read_int(batch_gen_fd);
 364                                 if (next_gen_i == -1)
 365                                         next_gen_i = flist->count;
 366 @@ -417,6 +426,7 @@ int recv_files(int f_in, struct file_lis
 367                                 discard_receive_data(f_in, file->length);
 368                                 continue;
 369                         }
 370 +                       next_gen_i = -1;
 371                 }
 372
 373                 if (server_exclude_list.head
 374 @@ -426,35 +436,32 @@ int recv_files(int f_in, struct file_lis
 375                         exit_cleanup(RERR_PROTOCOL);
 376                 }
 377
 378 -               if (partial_dir) {
 379 -                       if ((partialptr = partial_dir_fname(fname)) != NULL)
 380 -                               fnamecmp = partialptr;
 381 -                       else
 382 +               partialptr = partial_dir ? partial_dir_fname(fname) : fname;
 383 +
 384 +               if (f_in_name >= 0) {
 385 +                       switch (read_byte(f_in_name)) {
 386 +                       case FNAMECMP_FNAME:
 387                                 fnamecmp = fname;
 388 +                               break;
 389 +                       case FNAMECMP_PARTIAL_DIR:
 390 +                               fnamecmp = partialptr ? partialptr : fname;
 391 +                               break;
 392 +                       case FNAMECMP_BACKUP:
 393 +                               fnamecmp = get_backup_name(fname);
 394 +                               break;
 395 +                       case FNAMECMP_CMPDEST:
 396 +                       default:
 397 +                               pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 398 +                                        compare_dest, fname);
 399 +                               fnamecmp = fnamecmpbuf;
 400 +                               break;
 401 +                       }
 402                 } else
 403 -                       fnamecmp = partialptr = fname;
 404 -
 405 -               if (inplace && make_backups) {
 406 -                       if (!(fnamecmp = get_backup_name(fname)))
 407 -                               fnamecmp = partialptr;
 408 -               }
 409 +                       fnamecmp = fname;
 410
 411                 /* open the file */
 412                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 413
 414 -               if (fd1 == -1 && fnamecmp != fname) {
 415 -                       fnamecmp = fname;
 416 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 417 -               }
 418 -
 419 -               if (fd1 == -1 && compare_dest != NULL) {
 420 -                       /* try the file at compare_dest instead */
 421 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 422 -                                compare_dest, fname);
 423 -                       fnamecmp = fnamecmpbuf;
 424 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 425 -               }
 426 -
 427                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 428                         rsyserr(FERROR, errno, "fstat %s failed",
 429                                 full_fname(fnamecmp));
 430 --- orig/rsync.h        2004-10-09 03:21:56
 431 +++ rsync.h     2004-11-03 20:11:34
 432 @@ -119,6 +119,11 @@
 433  #define PDIR_CREATE    1
 434  #define PDIR_DELETE    0
 435
 436 +#define FNAMECMP_CMPDEST       0x00
 437 +#define FNAMECMP_FNAME         0x80
 438 +#define FNAMECMP_PARTIAL_DIR   0x81
 439 +#define FNAMECMP_BACKUP        0x82
 440 +
 441
 442  /* Log-message categories.  FLOG is only used on the daemon side to
 443   * output messages to the log file. */