g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 You must run "make proto" before compiling.
   9
  10 --- orig/generator.c    2004-07-23 17:16:12
  11 +++ generator.c 2004-07-22 00:05:38
  12 @@ -258,9 +258,9 @@ static void generate_and_send_sums(int f
  13   * out.  It might be wrong.
  14   */
  15  static void recv_generator(char *fname, struct file_struct *file, int i,
  16 -                          int f_out)
  17 +                          int f_out, int f_out_name)
  18  {
  19 -       int fd;
  20 +       int fd = -1;
  21         STRUCT_STAT st;
  22         int statret;
  23         char *fnamecmp;
  24 @@ -420,8 +420,10 @@ static void recv_generator(char *fname,
  25                 statret = link_stat(fnamecmpbuf, &st, 0);
  26                 if (!S_ISREG(st.st_mode))
  27                         statret = -1;
  28 -               if (statret == -1)
  29 +               if (statret == -1) {
  30                         errno = saveerrno;
  31 +                       *fnamecmpbuf = '\0';
  32 +               }
  33  #if HAVE_LINK
  34                 else if (link_dest && !dry_run) {
  35                         if (do_link(fnamecmpbuf, fname) != 0) {
  36 @@ -429,22 +431,22 @@ static void recv_generator(char *fname,
  37                                         rsyserr(FINFO, errno, "link %s => %s",
  38                                                 fnamecmpbuf, fname);
  39                                 }
  40 -                       }
  41 -                       fnamecmp = fnamecmpbuf;
  42 +                               fnamecmp = fnamecmpbuf;
  43 +                       } else
  44 +                               *fnamecmpbuf = '\0';
  45                 }
  46  #endif
  47                 else
  48                         fnamecmp = fnamecmpbuf;
  49 -       }
  50 +       } else
  51 +               *fnamecmpbuf = '\0';
  52
  53         if (statret == -1) {
  54                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  55                         return;
  56 -               if (errno == ENOENT) {
  57 -                       write_int(f_out,i);
  58 -                       if (!dry_run && !read_batch)
  59 -                               write_sum_head(f_out, NULL);
  60 -               } else if (verbose > 1) {
  61 +               if (errno == ENOENT)
  62 +                       goto notify_others;
  63 +               if (verbose > 1) {
  64                         rsyserr(FERROR, errno,
  65                                 "recv_generator: failed to open %s",
  66                                 full_fname(fname));
  67 @@ -453,26 +455,23 @@ static void recv_generator(char *fname,
  68         }
  69
  70         if (!S_ISREG(st.st_mode)) {
  71 -               if (delete_file(fname) != 0) {
  72 +               if (delete_file(fname) != 0)
  73                         return;
  74 -               }
  75
  76                 /* now pretend the file didn't exist */
  77                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  78                         return;
  79 -               write_int(f_out,i);
  80 -               if (!dry_run && !read_batch)
  81 -                       write_sum_head(f_out, NULL);
  82 -               return;
  83 +               statret = -1;
  84 +               goto notify_others;
  85         }
  86
  87 -       if (opt_ignore_existing && fnamecmp == fname) {
  88 +       if (opt_ignore_existing && !*fnamecmpbuf) {
  89                 if (verbose > 1)
  90                         rprintf(FINFO,"%s exists\n",fname);
  91                 return;
  92         }
  93
  94 -       if (update_only && fnamecmp == fname
  95 +       if (update_only && !*fnamecmpbuf
  96             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
  97                 if (verbose > 1)
  98                         rprintf(FINFO,"%s is newer\n",fname);
  99 @@ -480,21 +479,17 @@ static void recv_generator(char *fname,
 100         }
 101
 102         if (skip_file(fname, file, &st)) {
 103 -               if (fnamecmp == fname)
 104 +               if (!*fnamecmpbuf)
 105                         set_perms(fname, file, &st, PERMS_REPORT);
 106                 return;
 107         }
 108
 109 -       if (dry_run || read_batch) {
 110 -               write_int(f_out,i);
 111 -               return;
 112 -       }
 113 -
 114 -       if (whole_file > 0) {
 115 -               write_int(f_out,i);
 116 -               write_sum_head(f_out, NULL);
 117 -               return;
 118 +       if (dry_run || whole_file) {
 119 +               statret = -1;
 120 +               goto notify_others;
 121         }
 122 +       if (read_batch)
 123 +               goto notify_others;
 124
 125         /* open the file */
 126         fd = do_open(fnamecmp, O_RDONLY, 0);
 127 @@ -505,9 +500,8 @@ static void recv_generator(char *fname,
 128                 /* pretend the file didn't exist */
 129                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 130                         return;
 131 -               write_int(f_out,i);
 132 -               write_sum_head(f_out, NULL);
 133 -               return;
 134 +               statret = -1;
 135 +               goto notify_others;
 136         }
 137
 138         if (verbose > 3) {
 139 @@ -518,14 +512,41 @@ static void recv_generator(char *fname,
 140         if (verbose > 2)
 141                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 142
 143 -       write_int(f_out,i);
 144 -       generate_and_send_sums(fd, st.st_size, f_out);
 145 +notify_others:
 146 +       if (f_out_name >= 0) {
 147 +               uchar lenbuf[3], *lb = lenbuf;
 148 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 149 +               if (len > 0x7F) {
 150 +#if MAXPATHLEN > 0x7FFF
 151 +                       *lb++ = len / 0x10000 + 0x80;
 152 +                       *lb++ = len / 0x100;
 153 +#else
 154 +                       *lb++ = len / 0x100 + 0x80;
 155 +#endif
 156 +               }
 157 +               *lb = len;
 158 +               write_buf(f_out_name, lenbuf, lb - lenbuf + 1);
 159 +               if (len)
 160 +                       write_buf(f_out_name, fnamecmpbuf, len);
 161 +               io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */
 162 +       }
 163 +
 164 +       write_int(f_out, i);
 165 +
 166 +       if (dry_run || read_batch)
 167 +               return;
 168
 169 -       close(fd);
 170 +       if (statret == 0) {
 171 +               generate_and_send_sums(fd, st.st_size, f_out);
 172 +
 173 +               close(fd);
 174 +       } else
 175 +               write_sum_head(f_out, NULL);
 176  }
 177
 178
 179 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 180 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 181 +                   int f_out_name)
 182  {
 183         int i;
 184         int phase = 0;
 185 @@ -566,7 +587,7 @@ void generate_files(int f_out, struct fi
 186                 }
 187
 188                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 189 -                              file, i, f_out);
 190 +                              file, i, f_out, f_out_name);
 191         }
 192
 193         phase++;
 194 @@ -583,7 +604,7 @@ void generate_files(int f_out, struct fi
 195         while ((i = get_redo_num()) != -1) {
 196                 struct file_struct *file = flist->files[i];
 197                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 198 -                              file, i, f_out);
 199 +                              file, i, f_out, f_out_name);
 200         }
 201
 202         phase++;
 203 @@ -602,7 +623,7 @@ void generate_files(int f_out, struct fi
 204                 if (!file->basename || !S_ISDIR(file->mode))
 205                         continue;
 206                 recv_generator(local_name ? local_name : f_name(file),
 207 -                              file, i, -1);
 208 +                              file, i, -1, -1);
 209         }
 210
 211         if (verbose > 2)
 212 --- orig/main.c 2004-07-23 17:16:13
 213 +++ main.c      2004-07-22 00:10:43
 214 @@ -58,6 +58,7 @@ extern int filesfrom_fd;
 215  extern pid_t cleanup_child_pid;
 216  extern char *files_from;
 217  extern char *remote_filesfrom_file;
 218 +extern char *compare_dest;
 219  extern char *rsync_path;
 220  extern char *shell_cmd;
 221  extern char *batch_name;
 222 @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st
 223  {
 224         int pid;
 225         int status = 0;
 226 -       int error_pipe[2];
 227 +       int error_pipe[2], name_pipe[2];
 228 +       BOOL need_name_pipe = compare_dest && !dry_run;
 229
 230         if (preserve_hard_links)
 231                 init_hard_links(flist);
 232 @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st
 233                         delete_files(flist);
 234         }
 235
 236 -       if (fd_pair(error_pipe) < 0) {
 237 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 238 +       if (fd_pair(error_pipe) < 0
 239 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 240 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 241                 exit_cleanup(RERR_SOCKETIO);
 242         }
 243
 244 @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st
 245
 246         if ((pid = do_fork()) == 0) {
 247                 close(error_pipe[0]);
 248 +               if (need_name_pipe) {
 249 +                       close(name_pipe[1]);
 250 +                       set_blocking(name_pipe[0]);
 251 +               } else
 252 +                       name_pipe[0] = -1;
 253                 if (f_in != f_out)
 254                         close(f_out);
 255
 256 @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st
 257                 /* set place to send errors */
 258                 set_msg_fd_out(error_pipe[1]);
 259
 260 -               recv_files(f_in,flist,local_name);
 261 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 262                 io_flush(FULL_FLUSH);
 263                 report(f_in);
 264
 265 @@ -503,6 +511,11 @@ static int do_recv(int f_in,int f_out,st
 266                 stop_write_batch();
 267
 268         close(error_pipe[1]);
 269 +       if (need_name_pipe) {
 270 +               close(name_pipe[0]);
 271 +               set_nonblocking(name_pipe[1]);
 272 +       } else
 273 +               name_pipe[1] = -1;
 274         if (f_in != f_out)
 275                 close(f_in);
 276
 277 @@ -510,7 +523,7 @@ static int do_recv(int f_in,int f_out,st
 278
 279         set_msg_fd_in(error_pipe[0]);
 280
 281 -       generate_files(f_out, flist, local_name);
 282 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 283
 284         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 285         report(-1);
 286 --- orig/receiver.c     2004-07-23 17:16:13
 287 +++ receiver.c  2004-07-22 00:20:50
 288 @@ -320,6 +320,30 @@ static int receive_data(int f_in, char *
 289  }
 290
 291
 292 +static char *read_gen_name(int fd, char *buf, char *realname)
 293 +{
 294 +       int len = read_byte(fd);
 295 +       if (len & 0x80) {
 296 +#if MAXPATHLEN > 32767
 297 +               uchar lenbuf[2];
 298 +               read_buf(fd, (char *)lenbuf, 2);
 299 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 300 +#else
 301 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 302 +#endif
 303 +       }
 304 +       if (len) {
 305 +               if (len >= MAXPATHLEN) {
 306 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 307 +                       exit_cleanup(RERR_PROTOCOL);
 308 +               }
 309 +               read_sbuf(fd, buf, len);
 310 +               return buf;
 311 +       }
 312 +       return realname;
 313 +}
 314 +
 315 +
 316  static void discard_receive_data(int f_in, OFF_T length)
 317  {
 318         receive_data(f_in, NULL, -1, 0, NULL, -1, length);
 319 @@ -330,7 +354,8 @@ static void discard_receive_data(int f_i
 320   * main routine for receiver process.
 321   *
 322   * Receiver process runs on the same host as the generator process. */
 323 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 324 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 325 +              int f_in_name)
 326  {
 327         int next_gen_i = -1;
 328         int fd1,fd2;
 329 @@ -359,8 +384,15 @@ int recv_files(int f_in, struct file_lis
 330                 i = read_int(f_in);
 331                 if (i == -1) {
 332                         if (read_batch) {
 333 -                               if (next_gen_i != flist->count)
 334 -                                       while (read_int(batch_gen_fd) != -1) {}
 335 +                               while (next_gen_i < flist->count) {
 336 +                                       if (f_in_name >= 0 && next_gen_i >= 0) {
 337 +                                               read_gen_name(f_in_name,
 338 +                                                       fnamecmpbuf, NULL);
 339 +                                       }
 340 +                                       next_gen_i = read_int(batch_gen_fd);
 341 +                                       if (next_gen_i < 0)
 342 +                                               break;
 343 +                               }
 344                                 next_gen_i = -1;
 345                         }
 346
 347 @@ -406,10 +438,12 @@ int recv_files(int f_in, struct file_lis
 348                 if (verbose > 2)
 349                         rprintf(FINFO,"recv_files(%s)\n",fname);
 350
 351 -               fnamecmp = fname;
 352 -
 353                 if (read_batch) {
 354                         while (i > next_gen_i) {
 355 +                               if (f_in_name >= 0 && next_gen_i >= 0) {
 356 +                                       read_gen_name(f_in_name, fnamecmpbuf,
 357 +                                                     NULL);
 358 +                               }
 359                                 next_gen_i = read_int(batch_gen_fd);
 360                                 if (next_gen_i == -1)
 361                                         next_gen_i = flist->count;
 362 @@ -420,8 +454,14 @@ int recv_files(int f_in, struct file_lis
 363                                 discard_receive_data(f_in, file->length);
 364                                 continue;
 365                         }
 366 +                       next_gen_i = -1;
 367                 }
 368
 369 +               if (f_in_name >= 0)
 370 +                       fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname);
 371 +               else
 372 +                       fnamecmp = fname;
 373 +
 374                 if (server_exclude_list.head
 375                     && check_exclude(&server_exclude_list, fname,
 376                                      S_ISDIR(file->mode)) < 0) {
 377 @@ -437,14 +477,6 @@ int recv_files(int f_in, struct file_lis
 378                 /* open the file */
 379                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 380
 381 -               if (fd1 == -1 && compare_dest != NULL) {
 382 -                       /* try the file at compare_dest instead */
 383 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 384 -                                compare_dest, fname);
 385 -                       fnamecmp = fnamecmpbuf;
 386 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 387 -               }
 388 -
 389                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 390                         rsyserr(FERROR, errno, "fstat %s failed",
 391                                 full_fname(fnamecmp));