g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 Also fixes a potential synchronization problem between the generator
   9 and the receiver in read-batch mode.  Should consider making the
  10 sending of the index value the default for this pipe (it's currently
  11 only sent in batch mode due to the sender not listening to the
  12 generator to determine what files get updated).
  13
  14 You must run "make proto" before compiling.
  15
  16 --- orig/generator.c    2004-07-17 16:30:20
  17 +++ generator.c 2004-07-17 15:50:09
  18 @@ -251,11 +251,11 @@ static void generate_and_send_sums(struc
  19   * out.  It might be wrong.
  20   */
  21  static void recv_generator(char *fname, struct file_struct *file, int i,
  22 -                          int f_out)
  23 +                          int f_out, int f_nameout)
  24  {
  25 -       int fd;
  26 +       int fd = -1;
  27         STRUCT_STAT st;
  28 -       struct map_struct *mapbuf;
  29 +       struct map_struct *mapbuf = NULL;
  30         int statret;
  31         char *fnamecmp;
  32         char fnamecmpbuf[MAXPATHLEN];
  33 @@ -398,9 +398,6 @@ static void recv_generator(char *fname,
  34         }
  35  #endif
  36
  37 -       if (read_batch)
  38 -               return;
  39 -
  40         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
  41                 return;
  42
  43 @@ -418,8 +415,10 @@ static void recv_generator(char *fname,
  44                 statret = link_stat(fnamecmpbuf, &st, 0);
  45                 if (!S_ISREG(st.st_mode))
  46                         statret = -1;
  47 -               if (statret == -1)
  48 +               if (statret == -1) {
  49                         errno = saveerrno;
  50 +                       *fnamecmpbuf = '\0';
  51 +               }
  52  #if HAVE_LINK
  53                 else if (link_dest && !dry_run) {
  54                         if (do_link(fnamecmpbuf, fname) != 0) {
  55 @@ -427,22 +426,22 @@ static void recv_generator(char *fname,
  56                                         rsyserr(FINFO, errno, "link %s => %s",
  57                                                 fnamecmpbuf, fname);
  58                                 }
  59 -                       }
  60 -                       fnamecmp = fnamecmpbuf;
  61 +                               fnamecmp = fnamecmpbuf;
  62 +                       } else
  63 +                               *fnamecmpbuf = '\0';
  64                 }
  65  #endif
  66                 else
  67                         fnamecmp = fnamecmpbuf;
  68 -       }
  69 +       } else
  70 +               *fnamecmpbuf = '\0';
  71
  72         if (statret == -1) {
  73                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  74                         return;
  75 -               if (errno == ENOENT) {
  76 -                       write_int(f_out,i);
  77 -                       if (!dry_run)
  78 -                               write_sum_head(f_out, NULL);
  79 -               } else if (verbose > 1) {
  80 +               if (errno == ENOENT)
  81 +                       goto notify_sender;
  82 +               if (verbose > 1) {
  83                         rsyserr(FERROR, errno,
  84                                 "recv_generator: failed to open %s",
  85                                 full_fname(fname));
  86 @@ -451,26 +450,23 @@ static void recv_generator(char *fname,
  87         }
  88
  89         if (!S_ISREG(st.st_mode)) {
  90 -               if (delete_file(fname) != 0) {
  91 +               if (delete_file(fname) != 0)
  92                         return;
  93 -               }
  94
  95                 /* now pretend the file didn't exist */
  96                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  97                         return;
  98 -               write_int(f_out,i);
  99 -               if (!dry_run)
 100 -                       write_sum_head(f_out, NULL);
 101 -               return;
 102 +               statret = -1;
 103 +               goto notify_sender;
 104         }
 105
 106 -       if (opt_ignore_existing && fnamecmp == fname) {
 107 +       if (opt_ignore_existing && !*fnamecmpbuf) {
 108                 if (verbose > 1)
 109                         rprintf(FINFO,"%s exists\n",fname);
 110                 return;
 111         }
 112
 113 -       if (update_only && fnamecmp == fname
 114 +       if (update_only && !*fnamecmpbuf
 115             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 116                 if (verbose > 1)
 117                         rprintf(FINFO,"%s is newer\n",fname);
 118 @@ -478,21 +474,17 @@ static void recv_generator(char *fname,
 119         }
 120
 121         if (skip_file(fname, file, &st)) {
 122 -               if (fnamecmp == fname)
 123 +               if (!*fnamecmpbuf)
 124                         set_perms(fname, file, &st, PERMS_REPORT);
 125                 return;
 126         }
 127
 128 -       if (dry_run) {
 129 -               write_int(f_out,i);
 130 -               return;
 131 -       }
 132 -
 133 -       if (whole_file > 0) {
 134 -               write_int(f_out,i);
 135 -               write_sum_head(f_out, NULL);
 136 -               return;
 137 +       if (dry_run || whole_file) {
 138 +               statret = -1;
 139 +               goto notify_sender;
 140         }
 141 +       if (read_batch)
 142 +               goto notify_sender;
 143
 144         /* open the file */
 145         fd = do_open(fnamecmp, O_RDONLY, 0);
 146 @@ -503,15 +495,12 @@ static void recv_generator(char *fname,
 147                 /* pretend the file didn't exist */
 148                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 149                         return;
 150 -               write_int(f_out,i);
 151 -               write_sum_head(f_out, NULL);
 152 -               return;
 153 +               statret = -1;
 154 +               goto notify_sender;
 155         }
 156
 157         if (st.st_size > 0)
 158                 mapbuf = map_file(fd,st.st_size);
 159 -       else
 160 -               mapbuf = NULL;
 161
 162         if (verbose > 3) {
 163                 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
 164 @@ -521,16 +510,43 @@ static void recv_generator(char *fname,
 165         if (verbose > 2)
 166                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 167
 168 -       write_int(f_out,i);
 169 -       generate_and_send_sums(mapbuf, st.st_size, f_out);
 170 +notify_sender:
 171 +       if (f_nameout >= 0) {
 172 +               uchar lenbuf[3], *lb = lenbuf;
 173 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 174 +               if (read_batch)
 175 +                       write_int(f_nameout, i);
 176 +               if (len > 0x7F) {
 177 +#if MAXPATHLEN > 0x7FFF
 178 +                       *lb++ = len / 0x10000 + 0x80;
 179 +                       *lb++ = len / 0x100;
 180 +#else
 181 +                       *lb++ = len / 0x100 + 0x80;
 182 +#endif
 183 +               }
 184 +               *lb = len;
 185 +               write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
 186 +               if (len)
 187 +                       write_buf(f_nameout, fnamecmpbuf, len);
 188 +       }
 189
 190 -       close(fd);
 191 -       if (mapbuf)
 192 -               unmap_file(mapbuf);
 193 +       if (read_batch)
 194 +               return;
 195 +
 196 +       write_int(f_out, i);
 197 +       if (statret == 0) {
 198 +               generate_and_send_sums(mapbuf, st.st_size, f_out);
 199 +
 200 +               close(fd);
 201 +               if (mapbuf)
 202 +                       unmap_file(mapbuf);
 203 +       } else if (!dry_run)
 204 +               write_sum_head(f_out, NULL);
 205  }
 206
 207
 208 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 209 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 210 +                   int f_nameout)
 211  {
 212         int i;
 213         int phase = 0;
 214 @@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi
 215                 }
 216
 217                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 218 -                              file, i, f_out);
 219 +                              file, i, f_out, f_nameout);
 220         }
 221
 222         phase++;
 223 @@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi
 224                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 225
 226         write_int(f_out, -1);
 227 +       if (read_batch)
 228 +               write_int(f_nameout, flist->count);
 229
 230         /* files can cycle through the system more than once
 231          * to catch initial checksum errors */
 232         while ((i = get_redo_num()) != -1) {
 233                 struct file_struct *file = flist->files[i];
 234                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 235 -                              file, i, f_out);
 236 +                              file, i, f_out, f_nameout);
 237         }
 238
 239         phase++;
 240 @@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi
 241                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 242
 243         write_int(f_out, -1);
 244 +       if (read_batch)
 245 +               write_int(f_nameout, flist->count);
 246
 247         if (preserve_hard_links)
 248                 do_hard_links();
 249 @@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi
 250                 if (!file->basename || !S_ISDIR(file->mode))
 251                         continue;
 252                 recv_generator(local_name ? local_name : f_name(file),
 253 -                              file, i, -1);
 254 +                              file, i, -1, -1);
 255         }
 256
 257         if (verbose > 2)
 258 --- orig/main.c 2004-07-17 15:20:05
 259 +++ main.c      2004-07-17 15:58:11
 260 @@ -57,6 +57,7 @@ extern int filesfrom_fd;
 261  extern pid_t cleanup_child_pid;
 262  extern char *files_from;
 263  extern char *remote_filesfrom_file;
 264 +extern char *compare_dest;
 265  extern char *rsync_path;
 266  extern char *shell_cmd;
 267  extern char *batch_name;
 268 @@ -444,7 +445,8 @@ static int do_recv(int f_in,int f_out,st
 269  {
 270         int pid;
 271         int status = 0;
 272 -       int error_pipe[2];
 273 +       int error_pipe[2], name_pipe[2];
 274 +       int need_name_pipe = compare_dest || read_batch;
 275
 276         if (preserve_hard_links)
 277                 init_hard_links(flist);
 278 @@ -456,8 +458,9 @@ static int do_recv(int f_in,int f_out,st
 279                 }
 280         }
 281
 282 -       if (fd_pair(error_pipe) < 0) {
 283 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 284 +       if (fd_pair(error_pipe) < 0
 285 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 286 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 287                 exit_cleanup(RERR_SOCKETIO);
 288         }
 289
 290 @@ -465,6 +468,11 @@ static int do_recv(int f_in,int f_out,st
 291
 292         if ((pid = do_fork()) == 0) {
 293                 close(error_pipe[0]);
 294 +               if (need_name_pipe) {
 295 +                       close(name_pipe[1]);
 296 +                       set_blocking(name_pipe[0]);
 297 +               } else
 298 +                       name_pipe[0] = -1;
 299                 if (f_in != f_out)
 300                         close(f_out);
 301
 302 @@ -474,7 +482,7 @@ static int do_recv(int f_in,int f_out,st
 303                 /* set place to send errors */
 304                 set_msg_fd_out(error_pipe[1]);
 305
 306 -               recv_files(f_in,flist,local_name);
 307 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 308                 io_flush(FULL_FLUSH);
 309                 report(f_in);
 310
 311 @@ -492,6 +500,11 @@ static int do_recv(int f_in,int f_out,st
 312                 stop_write_batch();
 313
 314         close(error_pipe[1]);
 315 +       if (need_name_pipe) {
 316 +               close(name_pipe[0]);
 317 +               set_nonblocking(name_pipe[1]);
 318 +       } else
 319 +               name_pipe[1] = -1;
 320         if (f_in != f_out)
 321                 close(f_in);
 322
 323 @@ -499,7 +512,7 @@ static int do_recv(int f_in,int f_out,st
 324
 325         set_msg_fd_in(error_pipe[0]);
 326
 327 -       generate_files(f_out, flist, local_name);
 328 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 329
 330         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 331         report(-1);
 332 --- orig/receiver.c     2004-07-16 20:07:22
 333 +++ receiver.c  2004-07-17 21:27:55
 334 @@ -28,6 +28,7 @@ extern int max_delete;
 335  extern int csum_length;
 336  extern struct stats stats;
 337  extern int dry_run;
 338 +extern int read_batch;
 339  extern int am_server;
 340  extern int relative_paths;
 341  extern int keep_dirlinks;
 342 @@ -299,13 +300,38 @@ static int receive_data(int f_in,struct
 343         return 1;
 344  }
 345
 346 +static char *read_gen_name(int fd, char *buf, char *realname)
 347 +{
 348 +       int len = read_byte(fd);
 349 +       if (len & 0x80) {
 350 +#if MAXPATHLEN > 32767
 351 +               uchar lenbuf[2];
 352 +               read_buf(fd, (char *)lenbuf, 2);
 353 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 354 +#else
 355 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 356 +#endif
 357 +       }
 358 +       if (len) {
 359 +               if (len >= MAXPATHLEN) {
 360 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 361 +                       exit_cleanup(RERR_PROTOCOL);
 362 +               }
 363 +               read_sbuf(fd, buf, len);
 364 +               return buf;
 365 +       }
 366 +       return realname;
 367 +}
 368 +
 369
 370  /**
 371   * main routine for receiver process.
 372   *
 373   * Receiver process runs on the same host as the generator process. */
 374 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 375 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 376 +              int f_name_in)
 377  {
 378 +       int next_gen_i = -1;
 379         int fd1,fd2;
 380         STRUCT_STAT st;
 381         char *fname, fbuf[MAXPATHLEN];
 382 @@ -332,8 +358,20 @@ int recv_files(int f_in, struct file_lis
 383
 384                 i = read_int(f_in);
 385                 if (i == -1) {
 386 +                       if (read_batch) {
 387 +                               if (next_gen_i < 0)
 388 +                                       next_gen_i = read_int(f_name_in);
 389 +                               while (next_gen_i < flist->count) {
 390 +                                       read_gen_name(f_name_in, fnamecmpbuf,
 391 +                                                     NULL);
 392 +                                       next_gen_i = read_int(f_name_in);
 393 +                               }
 394 +                               next_gen_i = -1;
 395 +                       }
 396 +
 397                         if (phase)
 398                                 break;
 399 +
 400                         phase = 1;
 401                         csum_length = SUM_LENGTH;
 402                         if (verbose > 2)
 403 @@ -373,19 +411,31 @@ int recv_files(int f_in, struct file_lis
 404                 if (verbose > 2)
 405                         rprintf(FINFO,"recv_files(%s)\n",fname);
 406
 407 -               fnamecmp = fname;
 408 +               if (read_batch) {
 409 +                       if (next_gen_i < 0)
 410 +                               next_gen_i = read_int(f_name_in);
 411 +                       while (i > next_gen_i) {
 412 +                               read_gen_name(f_name_in, fnamecmpbuf, NULL);
 413 +                               next_gen_i = read_int(f_name_in);
 414 +                       }
 415 +                       if (i < next_gen_i) {
 416 +                               rprintf(FINFO, "skipping update for %s\n",
 417 +                                       fname);
 418 +                               receive_data(f_in,NULL,-1,NULL,file->length);
 419 +                               continue;
 420 +                       }
 421 +                       next_gen_i = -1;
 422 +               }
 423 +
 424 +               if (f_name_in >= 0)
 425 +                       fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
 426 +               else
 427 +                       fnamecmp = fname;
 428 +
 429
 430                 /* open the file */
 431                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 432
 433 -               if (fd1 == -1 && compare_dest != NULL) {
 434 -                       /* try the file at compare_dest instead */
 435 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 436 -                                compare_dest, fname);
 437 -                       fnamecmp = fnamecmpbuf;
 438 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 439 -               }
 440 -
 441                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 442                         rsyserr(FERROR, errno, "fstat %s failed",
 443                                 full_fname(fnamecmp));