g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 Also fixes a potential synchronization problem between the generator
   9 and the receiver in read-batch mode.  Should consider making the
  10 sending of the index value the default for this pipe (it's currently
  11 only sent in batch mode due to the sender not listening to the
  12 generator to determine what files get updated).
  13
  14 You must run "make proto" before compiling.
  15
  16 --- orig/generator.c    2004-07-17 16:30:20
  17 +++ generator.c 2004-07-17 15:50:09
  18 @@ -251,11 +251,11 @@ static void generate_and_send_sums(struc
  19   * out.  It might be wrong.
  20   */
  21  static void recv_generator(char *fname, struct file_struct *file, int i,
  22 -                          int f_out)
  23 +                          int f_out, int f_nameout)
  24  {
  25 -       int fd;
  26 +       int fd = -1;
  27         STRUCT_STAT st;
  28 -       struct map_struct *mapbuf;
  29 +       struct map_struct *mapbuf = NULL;
  30         int statret;
  31         char *fnamecmp;
  32         char fnamecmpbuf[MAXPATHLEN];
  33 @@ -398,9 +398,6 @@ static void recv_generator(char *fname,
  34         }
  35  #endif
  36
  37 -       if (read_batch)
  38 -               return;
  39 -
  40         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
  41                 return;
  42
  43 @@ -418,8 +415,10 @@ static void recv_generator(char *fname,
  44                 statret = link_stat(fnamecmpbuf, &st, 0);
  45                 if (!S_ISREG(st.st_mode))
  46                         statret = -1;
  47 -               if (statret == -1)
  48 +               if (statret == -1) {
  49                         errno = saveerrno;
  50 +                       *fnamecmpbuf = '\0';
  51 +               }
  52  #if HAVE_LINK
  53                 else if (link_dest && !dry_run) {
  54                         if (do_link(fnamecmpbuf, fname) != 0) {
  55 @@ -427,22 +426,22 @@ static void recv_generator(char *fname,
  56                                         rsyserr(FINFO, errno, "link %s => %s",
  57                                                 fnamecmpbuf, fname);
  58                                 }
  59 -                       }
  60 -                       fnamecmp = fnamecmpbuf;
  61 +                               fnamecmp = fnamecmpbuf;
  62 +                       } else
  63 +                               *fnamecmpbuf = '\0';
  64                 }
  65  #endif
  66                 else
  67                         fnamecmp = fnamecmpbuf;
  68 -       }
  69 +       } else
  70 +               *fnamecmpbuf = '\0';
  71
  72         if (statret == -1) {
  73                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  74                         return;
  75 -               if (errno == ENOENT) {
  76 -                       write_int(f_out,i);
  77 -                       if (!dry_run)
  78 -                               write_sum_head(f_out, NULL);
  79 -               } else if (verbose > 1) {
  80 +               if (errno == ENOENT)
  81 +                       goto notify_others;
  82 +               if (verbose > 1) {
  83                         rsyserr(FERROR, errno,
  84                                 "recv_generator: failed to open %s",
  85                                 full_fname(fname));
  86 @@ -451,26 +450,23 @@ static void recv_generator(char *fname,
  87         }
  88
  89         if (!S_ISREG(st.st_mode)) {
  90 -               if (delete_file(fname) != 0) {
  91 +               if (delete_file(fname) != 0)
  92                         return;
  93 -               }
  94
  95                 /* now pretend the file didn't exist */
  96                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  97                         return;
  98 -               write_int(f_out,i);
  99 -               if (!dry_run)
 100 -                       write_sum_head(f_out, NULL);
 101 -               return;
 102 +               statret = -1;
 103 +               goto notify_others;
 104         }
 105
 106 -       if (opt_ignore_existing && fnamecmp == fname) {
 107 +       if (opt_ignore_existing && !*fnamecmpbuf) {
 108                 if (verbose > 1)
 109                         rprintf(FINFO,"%s exists\n",fname);
 110                 return;
 111         }
 112
 113 -       if (update_only && fnamecmp == fname
 114 +       if (update_only && !*fnamecmpbuf
 115             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 116                 if (verbose > 1)
 117                         rprintf(FINFO,"%s is newer\n",fname);
 118 @@ -478,21 +474,17 @@ static void recv_generator(char *fname,
 119         }
 120
 121         if (skip_file(fname, file, &st)) {
 122 -               if (fnamecmp == fname)
 123 +               if (!*fnamecmpbuf)
 124                         set_perms(fname, file, &st, PERMS_REPORT);
 125                 return;
 126         }
 127
 128 -       if (dry_run) {
 129 -               write_int(f_out,i);
 130 -               return;
 131 -       }
 132 -
 133 -       if (whole_file > 0) {
 134 -               write_int(f_out,i);
 135 -               write_sum_head(f_out, NULL);
 136 -               return;
 137 +       if (dry_run || whole_file) {
 138 +               statret = -1;
 139 +               goto notify_others;
 140         }
 141 +       if (read_batch)
 142 +               goto notify_others;
 143
 144         /* open the file */
 145         fd = do_open(fnamecmp, O_RDONLY, 0);
 146 @@ -503,15 +495,12 @@ static void recv_generator(char *fname,
 147                 /* pretend the file didn't exist */
 148                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 149                         return;
 150 -               write_int(f_out,i);
 151 -               write_sum_head(f_out, NULL);
 152 -               return;
 153 +               statret = -1;
 154 +               goto notify_others;
 155         }
 156
 157         if (st.st_size > 0)
 158                 mapbuf = map_file(fd,st.st_size);
 159 -       else
 160 -               mapbuf = NULL;
 161
 162         if (verbose > 3) {
 163                 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
 164 @@ -521,16 +510,43 @@ static void recv_generator(char *fname,
 165         if (verbose > 2)
 166                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 167
 168 -       write_int(f_out,i);
 169 -       generate_and_send_sums(mapbuf, st.st_size, f_out);
 170 +notify_others:
 171 +       if (f_nameout >= 0) {
 172 +               uchar lenbuf[3], *lb = lenbuf;
 173 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 174 +               if (read_batch)
 175 +                       write_int(f_nameout, i);
 176 +               if (len > 0x7F) {
 177 +#if MAXPATHLEN > 0x7FFF
 178 +                       *lb++ = len / 0x10000 + 0x80;
 179 +                       *lb++ = len / 0x100;
 180 +#else
 181 +                       *lb++ = len / 0x100 + 0x80;
 182 +#endif
 183 +               }
 184 +               *lb = len;
 185 +               write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
 186 +               if (len)
 187 +                       write_buf(f_nameout, fnamecmpbuf, len);
 188 +       }
 189
 190 -       close(fd);
 191 -       if (mapbuf)
 192 -               unmap_file(mapbuf);
 193 +       if (read_batch)
 194 +               return;
 195 +
 196 +       write_int(f_out, i);
 197 +       if (statret == 0) {
 198 +               generate_and_send_sums(mapbuf, st.st_size, f_out);
 199 +
 200 +               close(fd);
 201 +               if (mapbuf)
 202 +                       unmap_file(mapbuf);
 203 +       } else if (!dry_run)
 204 +               write_sum_head(f_out, NULL);
 205  }
 206
 207
 208 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 209 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 210 +                   int f_nameout)
 211  {
 212         int i;
 213         int phase = 0;
 214 @@ -571,7 +587,7 @@ void generate_files(int f_out, struct fi
 215                 }
 216
 217                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 218 -                              file, i, f_out);
 219 +                              file, i, f_out, f_nameout);
 220         }
 221
 222         phase++;
 223 @@ -582,13 +598,15 @@ void generate_files(int f_out, struct fi
 224                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 225
 226         write_int(f_out, -1);
 227 +       if (read_batch)
 228 +               write_int(f_nameout, flist->count);
 229
 230         /* files can cycle through the system more than once
 231          * to catch initial checksum errors */
 232         while ((i = get_redo_num()) != -1) {
 233                 struct file_struct *file = flist->files[i];
 234                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 235 -                              file, i, f_out);
 236 +                              file, i, f_out, f_nameout);
 237         }
 238
 239         phase++;
 240 @@ -596,6 +614,8 @@ void generate_files(int f_out, struct fi
 241                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 242
 243         write_int(f_out, -1);
 244 +       if (read_batch)
 245 +               write_int(f_nameout, flist->count);
 246
 247         if (preserve_hard_links)
 248                 do_hard_links();
 249 @@ -607,7 +627,7 @@ void generate_files(int f_out, struct fi
 250                 if (!file->basename || !S_ISDIR(file->mode))
 251                         continue;
 252                 recv_generator(local_name ? local_name : f_name(file),
 253 -                              file, i, -1);
 254 +                              file, i, -1, -1);
 255         }
 256
 257         if (verbose > 2)
 258 --- orig/main.c 2004-07-19 17:14:44
 259 +++ main.c      2004-07-17 15:58:11
 260 @@ -57,6 +57,7 @@ extern int filesfrom_fd;
 261  extern pid_t cleanup_child_pid;
 262  extern char *files_from;
 263  extern char *remote_filesfrom_file;
 264 +extern char *compare_dest;
 265  extern char *rsync_path;
 266  extern char *shell_cmd;
 267  extern char *batch_name;
 268 @@ -444,20 +445,21 @@ static int do_recv(int f_in,int f_out,st
 269  {
 270         int pid;
 271         int status = 0;
 272 -       int error_pipe[2];
 273 +       int error_pipe[2], name_pipe[2];
 274 +       int need_name_pipe = compare_dest || read_batch;
 275
 276         if (preserve_hard_links)
 277                 init_hard_links(flist);
 278
 279         if (!delete_after) {
 280                 /* I moved this here from recv_files() to prevent a race condition */
 281 -               if (recurse && delete_mode && !local_name && flist->count>0) {
 282 +               if (recurse && delete_mode && !local_name && flist->count > 0)
 283                         delete_files(flist);
 284 -               }
 285         }
 286
 287 -       if (fd_pair(error_pipe) < 0) {
 288 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 289 +       if (fd_pair(error_pipe) < 0
 290 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 291 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 292                 exit_cleanup(RERR_SOCKETIO);
 293         }
 294
 295 @@ -465,6 +467,11 @@ static int do_recv(int f_in,int f_out,st
 296
 297         if ((pid = do_fork()) == 0) {
 298                 close(error_pipe[0]);
 299 +               if (need_name_pipe) {
 300 +                       close(name_pipe[1]);
 301 +                       set_blocking(name_pipe[0]);
 302 +               } else
 303 +                       name_pipe[0] = -1;
 304                 if (f_in != f_out)
 305                         close(f_out);
 306
 307 @@ -474,7 +481,7 @@ static int do_recv(int f_in,int f_out,st
 308                 /* set place to send errors */
 309                 set_msg_fd_out(error_pipe[1]);
 310
 311 -               recv_files(f_in,flist,local_name);
 312 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 313                 io_flush(FULL_FLUSH);
 314                 report(f_in);
 315
 316 @@ -492,6 +499,11 @@ static int do_recv(int f_in,int f_out,st
 317                 stop_write_batch();
 318
 319         close(error_pipe[1]);
 320 +       if (need_name_pipe) {
 321 +               close(name_pipe[0]);
 322 +               set_nonblocking(name_pipe[1]);
 323 +       } else
 324 +               name_pipe[1] = -1;
 325         if (f_in != f_out)
 326                 close(f_in);
 327
 328 @@ -499,7 +511,7 @@ static int do_recv(int f_in,int f_out,st
 329
 330         set_msg_fd_in(error_pipe[0]);
 331
 332 -       generate_files(f_out, flist, local_name);
 333 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 334
 335         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 336         report(-1);
 337 --- orig/receiver.c     2004-07-19 17:06:10
 338 +++ receiver.c  2004-07-19 16:44:39
 339 @@ -28,6 +28,7 @@ extern int max_delete;
 340  extern int csum_length;
 341  extern struct stats stats;
 342  extern int dry_run;
 343 +extern int read_batch;
 344  extern int am_server;
 345  extern int relative_paths;
 346  extern int keep_dirlinks;
 347 @@ -303,6 +304,30 @@ static int receive_data(int f_in,struct
 348  }
 349
 350
 351 +static char *read_gen_name(int fd, char *buf, char *realname)
 352 +{
 353 +       int len = read_byte(fd);
 354 +       if (len & 0x80) {
 355 +#if MAXPATHLEN > 32767
 356 +               uchar lenbuf[2];
 357 +               read_buf(fd, (char *)lenbuf, 2);
 358 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 359 +#else
 360 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 361 +#endif
 362 +       }
 363 +       if (len) {
 364 +               if (len >= MAXPATHLEN) {
 365 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 366 +                       exit_cleanup(RERR_PROTOCOL);
 367 +               }
 368 +               read_sbuf(fd, buf, len);
 369 +               return buf;
 370 +       }
 371 +       return realname;
 372 +}
 373 +
 374 +
 375  static void discard_receive_data(int f_in, OFF_T length)
 376  {
 377         receive_data(f_in, NULL, -1, NULL, length);
 378 @@ -313,8 +338,10 @@ static void discard_receive_data(int f_i
 379   * main routine for receiver process.
 380   *
 381   * Receiver process runs on the same host as the generator process. */
 382 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 383 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 384 +              int f_name_in)
 385  {
 386 +       int next_gen_i = -1;
 387         int fd1,fd2;
 388         STRUCT_STAT st;
 389         char *fname, fbuf[MAXPATHLEN];
 390 @@ -341,6 +368,17 @@ int recv_files(int f_in, struct file_lis
 391
 392                 i = read_int(f_in);
 393                 if (i == -1) {
 394 +                       if (read_batch) {
 395 +                               if (next_gen_i < 0)
 396 +                                       next_gen_i = read_int(f_name_in);
 397 +                               while (next_gen_i < flist->count) {
 398 +                                       read_gen_name(f_name_in, fnamecmpbuf,
 399 +                                                     NULL);
 400 +                                       next_gen_i = read_int(f_name_in);
 401 +                               }
 402 +                               next_gen_i = -1;
 403 +                       }
 404 +
 405                         if (phase)
 406                                 break;
 407
 408 @@ -383,7 +421,26 @@ int recv_files(int f_in, struct file_lis
 409                 if (verbose > 2)
 410                         rprintf(FINFO,"recv_files(%s)\n",fname);
 411
 412 -               fnamecmp = fname;
 413 +               if (read_batch) {
 414 +                       if (next_gen_i < 0)
 415 +                               next_gen_i = read_int(f_name_in);
 416 +                       while (i > next_gen_i) {
 417 +                               read_gen_name(f_name_in, fnamecmpbuf, NULL);
 418 +                               next_gen_i = read_int(f_name_in);
 419 +                       }
 420 +                       if (i < next_gen_i) {
 421 +                               rprintf(FINFO, "skipping update for \"%s\"\n",
 422 +                                       fname);
 423 +                               discard_receive_data(f_in, file->length);
 424 +                               continue;
 425 +                       }
 426 +                       next_gen_i = -1;
 427 +               }
 428 +
 429 +               if (f_name_in >= 0)
 430 +                       fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
 431 +               else
 432 +                       fnamecmp = fname;
 433
 434                 if (server_exclude_list.head
 435                     && check_exclude(&server_exclude_list, fname,
 436 @@ -400,14 +457,6 @@ int recv_files(int f_in, struct file_lis
 437                 /* open the file */
 438                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 439
 440 -               if (fd1 == -1 && compare_dest != NULL) {
 441 -                       /* try the file at compare_dest instead */
 442 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 443 -                                compare_dest, fname);
 444 -                       fnamecmp = fnamecmpbuf;
 445 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 446 -               }
 447 -
 448                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 449                         rsyserr(FERROR, errno, "fstat %s failed",
 450                                 full_fname(fnamecmp));