g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 You must run "make proto" before compiling.
   9
  10 --- orig/generator.c    2004-07-21 23:59:35
  11 +++ generator.c 2004-07-22 00:05:38
  12 @@ -258,9 +258,9 @@ static void generate_and_send_sums(int f
  13   * out.  It might be wrong.
  14   */
  15  static void recv_generator(char *fname, struct file_struct *file, int i,
  16 -                          int f_out)
  17 +                          int f_out, int f_nameout)
  18  {
  19 -       int fd;
  20 +       int fd = -1;
  21         STRUCT_STAT st;
  22         int statret;
  23         char *fnamecmp;
  24 @@ -421,8 +421,10 @@ static void recv_generator(char *fname,
  25                 statret = link_stat(fnamecmpbuf, &st, 0);
  26                 if (!S_ISREG(st.st_mode))
  27                         statret = -1;
  28 -               if (statret == -1)
  29 +               if (statret == -1) {
  30                         errno = saveerrno;
  31 +                       *fnamecmpbuf = '\0';
  32 +               }
  33  #if HAVE_LINK
  34                 else if (link_dest && !dry_run) {
  35                         if (do_link(fnamecmpbuf, fname) != 0) {
  36 @@ -430,22 +432,22 @@ static void recv_generator(char *fname,
  37                                         rsyserr(FINFO, errno, "link %s => %s",
  38                                                 fnamecmpbuf, fname);
  39                                 }
  40 -                       }
  41 -                       fnamecmp = fnamecmpbuf;
  42 +                               fnamecmp = fnamecmpbuf;
  43 +                       } else
  44 +                               *fnamecmpbuf = '\0';
  45                 }
  46  #endif
  47                 else
  48                         fnamecmp = fnamecmpbuf;
  49 -       }
  50 +       } else
  51 +               *fnamecmpbuf = '\0';
  52
  53         if (statret == -1) {
  54                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  55                         return;
  56 -               if (errno == ENOENT) {
  57 -                       write_int(f_out,i);
  58 -                       if (!dry_run && !read_batch)
  59 -                               write_sum_head(f_out, NULL);
  60 -               } else if (verbose > 1) {
  61 +               if (errno == ENOENT)
  62 +                       goto notify_others;
  63 +               if (verbose > 1) {
  64                         rsyserr(FERROR, errno,
  65                                 "recv_generator: failed to open %s",
  66                                 full_fname(fname));
  67 @@ -454,26 +456,23 @@ static void recv_generator(char *fname,
  68         }
  69
  70         if (!S_ISREG(st.st_mode)) {
  71 -               if (delete_file(fname) != 0) {
  72 +               if (delete_file(fname) != 0)
  73                         return;
  74 -               }
  75
  76                 /* now pretend the file didn't exist */
  77                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  78                         return;
  79 -               write_int(f_out,i);
  80 -               if (!dry_run && !read_batch)
  81 -                       write_sum_head(f_out, NULL);
  82 -               return;
  83 +               statret = -1;
  84 +               goto notify_others;
  85         }
  86
  87 -       if (opt_ignore_existing && fnamecmp == fname) {
  88 +       if (opt_ignore_existing && !*fnamecmpbuf) {
  89                 if (verbose > 1)
  90                         rprintf(FINFO,"%s exists\n",fname);
  91                 return;
  92         }
  93
  94 -       if (update_only && fnamecmp == fname
  95 +       if (update_only && !*fnamecmpbuf
  96             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
  97                 if (verbose > 1)
  98                         rprintf(FINFO,"%s is newer\n",fname);
  99 @@ -481,21 +480,17 @@ static void recv_generator(char *fname,
 100         }
 101
 102         if (skip_file(fname, file, &st)) {
 103 -               if (fnamecmp == fname)
 104 +               if (!*fnamecmpbuf)
 105                         set_perms(fname, file, &st, PERMS_REPORT);
 106                 return;
 107         }
 108
 109 -       if (dry_run || read_batch) {
 110 -               write_int(f_out,i);
 111 -               return;
 112 -       }
 113 -
 114 -       if (whole_file > 0) {
 115 -               write_int(f_out,i);
 116 -               write_sum_head(f_out, NULL);
 117 -               return;
 118 +       if (dry_run || whole_file) {
 119 +               statret = -1;
 120 +               goto notify_others;
 121         }
 122 +       if (read_batch)
 123 +               goto notify_others;
 124
 125         /* open the file */
 126         fd = do_open(fnamecmp, O_RDONLY, 0);
 127 @@ -506,9 +501,8 @@ static void recv_generator(char *fname,
 128                 /* pretend the file didn't exist */
 129                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 130                         return;
 131 -               write_int(f_out,i);
 132 -               write_sum_head(f_out, NULL);
 133 -               return;
 134 +               statret = -1;
 135 +               goto notify_others;
 136         }
 137
 138         if (verbose > 3) {
 139 @@ -519,14 +513,40 @@ static void recv_generator(char *fname,
 140         if (verbose > 2)
 141                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 142
 143 -       write_int(f_out,i);
 144 -       generate_and_send_sums(fd, st.st_size, f_out);
 145 +notify_others:
 146 +       if (f_nameout >= 0) {
 147 +               uchar lenbuf[3], *lb = lenbuf;
 148 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 149 +               if (len > 0x7F) {
 150 +#if MAXPATHLEN > 0x7FFF
 151 +                       *lb++ = len / 0x10000 + 0x80;
 152 +                       *lb++ = len / 0x100;
 153 +#else
 154 +                       *lb++ = len / 0x100 + 0x80;
 155 +#endif
 156 +               }
 157 +               *lb = len;
 158 +               write_buf(f_nameout, lenbuf, lb - lenbuf + 1);
 159 +               if (len)
 160 +                       write_buf(f_nameout, fnamecmpbuf, len);
 161 +       }
 162 +
 163 +       write_int(f_out, i);
 164 +
 165 +       if (read_batch)
 166 +               return;
 167
 168 -       close(fd);
 169 +       if (statret == 0) {
 170 +               generate_and_send_sums(fd, st.st_size, f_out);
 171 +
 172 +               close(fd);
 173 +       } else if (!dry_run)
 174 +               write_sum_head(f_out, NULL);
 175  }
 176
 177
 178 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 179 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 180 +                   int f_nameout)
 181  {
 182         int i;
 183         int phase = 0;
 184 @@ -567,7 +587,7 @@ void generate_files(int f_out, struct fi
 185                 }
 186
 187                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 188 -                              file, i, f_out);
 189 +                              file, i, f_out, f_nameout);
 190         }
 191
 192         phase++;
 193 @@ -584,7 +604,7 @@ void generate_files(int f_out, struct fi
 194         while ((i = get_redo_num()) != -1) {
 195                 struct file_struct *file = flist->files[i];
 196                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 197 -                              file, i, f_out);
 198 +                              file, i, f_out, f_nameout);
 199         }
 200
 201         phase++;
 202 @@ -603,7 +623,7 @@ void generate_files(int f_out, struct fi
 203                 if (!file->basename || !S_ISDIR(file->mode))
 204                         continue;
 205                 recv_generator(local_name ? local_name : f_name(file),
 206 -                              file, i, -1);
 207 +                              file, i, -1, -1);
 208         }
 209
 210         if (verbose > 2)
 211 --- orig/main.c 2004-07-21 23:59:35
 212 +++ main.c      2004-07-22 00:10:43
 213 @@ -58,6 +58,7 @@ extern int filesfrom_fd;
 214  extern pid_t cleanup_child_pid;
 215  extern char *files_from;
 216  extern char *remote_filesfrom_file;
 217 +extern char *compare_dest;
 218  extern char *rsync_path;
 219  extern char *shell_cmd;
 220  extern char *batch_name;
 221 @@ -454,20 +455,21 @@ static int do_recv(int f_in,int f_out,st
 222  {
 223         int pid;
 224         int status = 0;
 225 -       int error_pipe[2];
 226 +       int error_pipe[2], name_pipe[2];
 227 +       int need_name_pipe = compare_dest;
 228
 229         if (preserve_hard_links)
 230                 init_hard_links(flist);
 231
 232         if (!delete_after) {
 233                 /* I moved this here from recv_files() to prevent a race condition */
 234 -               if (recurse && delete_mode && !local_name && flist->count>0) {
 235 +               if (recurse && delete_mode && !local_name && flist->count > 0)
 236                         delete_files(flist);
 237 -               }
 238         }
 239
 240 -       if (fd_pair(error_pipe) < 0) {
 241 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 242 +       if (fd_pair(error_pipe) < 0
 243 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 244 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 245                 exit_cleanup(RERR_SOCKETIO);
 246         }
 247
 248 @@ -475,6 +477,11 @@ static int do_recv(int f_in,int f_out,st
 249
 250         if ((pid = do_fork()) == 0) {
 251                 close(error_pipe[0]);
 252 +               if (need_name_pipe) {
 253 +                       close(name_pipe[1]);
 254 +                       set_blocking(name_pipe[0]);
 255 +               } else
 256 +                       name_pipe[0] = -1;
 257                 if (f_in != f_out)
 258                         close(f_out);
 259
 260 @@ -484,7 +491,7 @@ static int do_recv(int f_in,int f_out,st
 261                 /* set place to send errors */
 262                 set_msg_fd_out(error_pipe[1]);
 263
 264 -               recv_files(f_in,flist,local_name);
 265 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 266                 io_flush(FULL_FLUSH);
 267                 report(f_in);
 268
 269 @@ -502,6 +509,11 @@ static int do_recv(int f_in,int f_out,st
 270                 stop_write_batch();
 271
 272         close(error_pipe[1]);
 273 +       if (need_name_pipe) {
 274 +               close(name_pipe[0]);
 275 +               set_nonblocking(name_pipe[1]);
 276 +       } else
 277 +               name_pipe[1] = -1;
 278         if (f_in != f_out)
 279                 close(f_in);
 280
 281 @@ -509,7 +521,7 @@ static int do_recv(int f_in,int f_out,st
 282
 283         set_msg_fd_in(error_pipe[0]);
 284
 285 -       generate_files(f_out, flist, local_name);
 286 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 287
 288         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 289         report(-1);
 290 --- orig/receiver.c     2004-07-22 00:07:20
 291 +++ receiver.c  2004-07-22 00:20:50
 292 @@ -320,6 +320,30 @@ static int receive_data(int f_in, char *
 293  }
 294
 295
 296 +static char *read_gen_name(int fd, char *buf, char *realname)
 297 +{
 298 +       int len = read_byte(fd);
 299 +       if (len & 0x80) {
 300 +#if MAXPATHLEN > 32767
 301 +               uchar lenbuf[2];
 302 +               read_buf(fd, (char *)lenbuf, 2);
 303 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 304 +#else
 305 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 306 +#endif
 307 +       }
 308 +       if (len) {
 309 +               if (len >= MAXPATHLEN) {
 310 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 311 +                       exit_cleanup(RERR_PROTOCOL);
 312 +               }
 313 +               read_sbuf(fd, buf, len);
 314 +               return buf;
 315 +       }
 316 +       return realname;
 317 +}
 318 +
 319 +
 320  static void discard_receive_data(int f_in, OFF_T length)
 321  {
 322         receive_data(f_in, NULL, -1, 0, NULL, -1, length);
 323 @@ -330,7 +354,8 @@ static void discard_receive_data(int f_i
 324   * main routine for receiver process.
 325   *
 326   * Receiver process runs on the same host as the generator process. */
 327 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 328 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 329 +              int f_name_in)
 330  {
 331         int next_gen_i = -1;
 332         int fd1,fd2;
 333 @@ -359,8 +384,15 @@ int recv_files(int f_in, struct file_lis
 334                 i = read_int(f_in);
 335                 if (i == -1) {
 336                         if (read_batch) {
 337 -                               if (next_gen_i != flist->count)
 338 -                                       while (read_int(batch_gen_fd) != -1) {}
 339 +                               while (next_gen_i < flist->count) {
 340 +                                       if (f_name_in >= 0 && next_gen_i >= 0) {
 341 +                                               read_gen_name(f_name_in,
 342 +                                                       fnamecmpbuf, NULL);
 343 +                                       }
 344 +                                       next_gen_i = read_int(f_name_in);
 345 +                                       if (next_gen_i < 0)
 346 +                                               break;
 347 +                               }
 348                                 next_gen_i = -1;
 349                         }
 350
 351 @@ -406,10 +438,12 @@ int recv_files(int f_in, struct file_lis
 352                 if (verbose > 2)
 353                         rprintf(FINFO,"recv_files(%s)\n",fname);
 354
 355 -               fnamecmp = fname;
 356 -
 357                 if (read_batch) {
 358                         while (i > next_gen_i) {
 359 +                               if (f_name_in >= 0 && next_gen_i >= 0) {
 360 +                                       read_gen_name(f_name_in, fnamecmpbuf,
 361 +                                                     NULL);
 362 +                               }
 363                                 next_gen_i = read_int(batch_gen_fd);
 364                                 if (next_gen_i == -1)
 365                                         next_gen_i = flist->count;
 366 @@ -420,8 +454,14 @@ int recv_files(int f_in, struct file_lis
 367                                 discard_receive_data(f_in, file->length);
 368                                 continue;
 369                         }
 370 +                       next_gen_i = -1;
 371                 }
 372
 373 +               if (f_name_in >= 0)
 374 +                       fnamecmp = read_gen_name(f_name_in, fnamecmpbuf, fname);
 375 +               else
 376 +                       fnamecmp = fname;
 377 +
 378                 if (server_exclude_list.head
 379                     && check_exclude(&server_exclude_list, fname,
 380                                      S_ISDIR(file->mode)) < 0) {
 381 @@ -437,14 +477,6 @@ int recv_files(int f_in, struct file_lis
 382                 /* open the file */
 383                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 384
 385 -               if (fd1 == -1 && compare_dest != NULL) {
 386 -                       /* try the file at compare_dest instead */
 387 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 388 -                                compare_dest, fname);
 389 -                       fnamecmp = fnamecmpbuf;
 390 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 391 -               }
 392 -
 393                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 394                         rsyserr(FERROR, errno, "fstat %s failed",
 395                                 full_fname(fnamecmp));