g2r-basis-filename.diff

   1 Added a pipe from the generator to the receiver that communicates
   2 what basis file we used to generate the file data (if it was not
   3 the default name).  This optimizes away the basis-file search in
   4 the receiver and makes future options that do more basis-file
   5 searching more efficient (such as the --fuzzy option and the
   6 support for multiple --compare-dest options).
   7
   8 You must run "make proto" before compiling.
   9
  10 --- orig/generator.c    2004-07-29 16:08:03
  11 +++ generator.c 2004-07-28 10:14:15
  12 @@ -259,9 +259,9 @@ static void generate_and_send_sums(int f
  13   * out.  It might be wrong.
  14   */
  15  static void recv_generator(char *fname, struct file_struct *file, int i,
  16 -                          int f_out)
  17 +                          int f_out, int f_out_name)
  18  {
  19 -       int fd;
  20 +       int fd = -1;
  21         STRUCT_STAT st;
  22         int statret, stat_errno;
  23         char *fnamecmp;
  24 @@ -437,13 +437,16 @@ static void recv_generator(char *fname,
  25                                                         safe_fname(fname));
  26                                         }
  27                                         fnamecmp = fnamecmpbuf;
  28 -                               }
  29 +                               } else
  30 +                                       *fnamecmpbuf = '\0';
  31                         } else
  32  #endif
  33                                 fnamecmp = fnamecmpbuf;
  34                         statret = 0;
  35 -               }
  36 -       }
  37 +               } else
  38 +                       *fnamecmpbuf = '\0';
  39 +       } else
  40 +               *fnamecmpbuf = '\0';
  41
  42         if (statret == 0 && !S_ISREG(st.st_mode)) {
  43                 if (delete_file(fname) != 0)
  44 @@ -455,11 +458,9 @@ static void recv_generator(char *fname,
  45         if (statret == -1) {
  46                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  47                         return;
  48 -               if (stat_errno == ENOENT) {
  49 -                       write_int(f_out,i);
  50 -                       if (!dry_run && !read_batch)
  51 -                               write_sum_head(f_out, NULL);
  52 -               } else if (verbose > 1) {
  53 +               if (stat_errno == ENOENT)
  54 +                       goto notify_others;
  55 +               if (verbose > 1) {
  56                         rsyserr(FERROR, stat_errno,
  57                                 "recv_generator: failed to stat %s",
  58                                 full_fname(fname));
  59 @@ -467,13 +468,13 @@ static void recv_generator(char *fname,
  60                 return;
  61         }
  62
  63 -       if (opt_ignore_existing && fnamecmp == fname) {
  64 +       if (opt_ignore_existing && !*fnamecmpbuf) {
  65                 if (verbose > 1)
  66                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
  67                 return;
  68         }
  69
  70 -       if (update_only && fnamecmp == fname
  71 +       if (update_only && !*fnamecmpbuf
  72             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
  73                 if (verbose > 1)
  74                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
  75 @@ -481,21 +482,17 @@ static void recv_generator(char *fname,
  76         }
  77
  78         if (skip_file(fname, file, &st)) {
  79 -               if (fnamecmp == fname)
  80 +               if (!*fnamecmpbuf)
  81                         set_perms(fname, file, &st, PERMS_REPORT);
  82                 return;
  83         }
  84
  85 -       if (dry_run || read_batch) {
  86 -               write_int(f_out,i);
  87 -               return;
  88 -       }
  89 -
  90 -       if (whole_file > 0) {
  91 -               write_int(f_out,i);
  92 -               write_sum_head(f_out, NULL);
  93 -               return;
  94 +       if (dry_run || whole_file) {
  95 +               statret = -1;
  96 +               goto notify_others;
  97         }
  98 +       if (read_batch)
  99 +               goto notify_others;
 100
 101         if (partial_dir) {
 102                 STRUCT_STAT st2;
 103 @@ -516,9 +513,8 @@ static void recv_generator(char *fname,
 104                 /* pretend the file didn't exist */
 105                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 106                         return;
 107 -               write_int(f_out,i);
 108 -               write_sum_head(f_out, NULL);
 109 -               return;
 110 +               statret = -1;
 111 +               goto notify_others;
 112         }
 113
 114         if (verbose > 3) {
 115 @@ -529,14 +525,41 @@ static void recv_generator(char *fname,
 116         if (verbose > 2)
 117                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 118
 119 -       write_int(f_out,i);
 120 -       generate_and_send_sums(fd, st.st_size, f_out);
 121 +notify_others:
 122 +       if (f_out_name >= 0) {
 123 +               uchar lenbuf[3], *lb = lenbuf;
 124 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 125 +               if (len > 0x7F) {
 126 +#if MAXPATHLEN > 0x7FFF
 127 +                       *lb++ = len / 0x10000 + 0x80;
 128 +                       *lb++ = len / 0x100;
 129 +#else
 130 +                       *lb++ = len / 0x100 + 0x80;
 131 +#endif
 132 +               }
 133 +               *lb = len;
 134 +               write_buf(f_out_name, lenbuf, lb - lenbuf + 1);
 135 +               if (len)
 136 +                       write_buf(f_out_name, fnamecmpbuf, len);
 137 +               io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */
 138 +       }
 139 +
 140 +       write_int(f_out, i);
 141 +
 142 +       if (dry_run || read_batch)
 143 +               return;
 144
 145 -       close(fd);
 146 +       if (statret == 0) {
 147 +               generate_and_send_sums(fd, st.st_size, f_out);
 148 +
 149 +               close(fd);
 150 +       } else
 151 +               write_sum_head(f_out, NULL);
 152  }
 153
 154
 155 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 156 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 157 +                   int f_out_name)
 158  {
 159         int i;
 160         int phase = 0;
 161 @@ -577,7 +600,7 @@ void generate_files(int f_out, struct fi
 162                 }
 163
 164                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 165 -                              file, i, f_out);
 166 +                              file, i, f_out, f_out_name);
 167         }
 168
 169         phase++;
 170 @@ -594,7 +617,7 @@ void generate_files(int f_out, struct fi
 171         while ((i = get_redo_num()) != -1) {
 172                 struct file_struct *file = flist->files[i];
 173                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 174 -                              file, i, f_out);
 175 +                              file, i, f_out, f_out_name);
 176         }
 177
 178         phase++;
 179 @@ -613,7 +636,7 @@ void generate_files(int f_out, struct fi
 180                 if (!file->basename || !S_ISDIR(file->mode))
 181                         continue;
 182                 recv_generator(local_name ? local_name : f_name(file),
 183 -                              file, i, -1);
 184 +                              file, i, -1, -1);
 185         }
 186
 187         if (verbose > 2)
 188 --- orig/main.c 2004-07-29 16:08:03
 189 +++ main.c      2004-07-22 00:10:43
 190 @@ -58,6 +58,7 @@ extern int filesfrom_fd;
 191  extern pid_t cleanup_child_pid;
 192  extern char *files_from;
 193  extern char *remote_filesfrom_file;
 194 +extern char *compare_dest;
 195  extern char *rsync_path;
 196  extern char *shell_cmd;
 197  extern char *batch_name;
 198 @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st
 199  {
 200         int pid;
 201         int status = 0;
 202 -       int error_pipe[2];
 203 +       int error_pipe[2], name_pipe[2];
 204 +       BOOL need_name_pipe = compare_dest && !dry_run;
 205
 206         if (preserve_hard_links)
 207                 init_hard_links(flist);
 208 @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st
 209                         delete_files(flist);
 210         }
 211
 212 -       if (fd_pair(error_pipe) < 0) {
 213 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 214 +       if (fd_pair(error_pipe) < 0
 215 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 216 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 217                 exit_cleanup(RERR_SOCKETIO);
 218         }
 219
 220 @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st
 221
 222         if ((pid = do_fork()) == 0) {
 223                 close(error_pipe[0]);
 224 +               if (need_name_pipe) {
 225 +                       close(name_pipe[1]);
 226 +                       set_blocking(name_pipe[0]);
 227 +               } else
 228 +                       name_pipe[0] = -1;
 229                 if (f_in != f_out)
 230                         close(f_out);
 231
 232 @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st
 233                 /* set place to send errors */
 234                 set_msg_fd_out(error_pipe[1]);
 235
 236 -               recv_files(f_in,flist,local_name);
 237 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 238                 io_flush(FULL_FLUSH);
 239                 report(f_in);
 240
 241 @@ -504,6 +512,11 @@ static int do_recv(int f_in,int f_out,st
 242                 stop_write_batch();
 243
 244         close(error_pipe[1]);
 245 +       if (need_name_pipe) {
 246 +               close(name_pipe[0]);
 247 +               set_nonblocking(name_pipe[1]);
 248 +       } else
 249 +               name_pipe[1] = -1;
 250         if (f_in != f_out)
 251                 close(f_in);
 252
 253 @@ -511,7 +524,7 @@ static int do_recv(int f_in,int f_out,st
 254
 255         set_msg_fd_in(error_pipe[0]);
 256
 257 -       generate_files(f_out, flist, local_name);
 258 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 259
 260         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 261         report(-1);
 262 --- orig/receiver.c     2004-07-29 16:08:03
 263 +++ receiver.c  2004-07-23 21:59:07
 264 @@ -323,6 +323,30 @@ static int receive_data(int f_in, char *
 265  }
 266
 267
 268 +static char *read_gen_name(int fd, char *buf, char *realname)
 269 +{
 270 +       int len = read_byte(fd);
 271 +       if (len & 0x80) {
 272 +#if MAXPATHLEN > 32767
 273 +               uchar lenbuf[2];
 274 +               read_buf(fd, (char *)lenbuf, 2);
 275 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 276 +#else
 277 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 278 +#endif
 279 +       }
 280 +       if (len) {
 281 +               if (len >= MAXPATHLEN) {
 282 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 283 +                       exit_cleanup(RERR_PROTOCOL);
 284 +               }
 285 +               read_sbuf(fd, buf, len);
 286 +               return buf;
 287 +       }
 288 +       return realname;
 289 +}
 290 +
 291 +
 292  static void discard_receive_data(int f_in, OFF_T length)
 293  {
 294         receive_data(f_in, NULL, -1, 0, NULL, -1, length);
 295 @@ -333,7 +357,8 @@ static void discard_receive_data(int f_i
 296   * main routine for receiver process.
 297   *
 298   * Receiver process runs on the same host as the generator process. */
 299 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 300 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 301 +              int f_in_name)
 302  {
 303         int next_gen_i = -1;
 304         int fd1,fd2;
 305 @@ -362,8 +387,15 @@ int recv_files(int f_in, struct file_lis
 306                 i = read_int(f_in);
 307                 if (i == -1) {
 308                         if (read_batch) {
 309 -                               if (next_gen_i != flist->count)
 310 -                                       while (read_int(batch_gen_fd) != -1) {}
 311 +                               while (next_gen_i < flist->count) {
 312 +                                       if (f_in_name >= 0 && next_gen_i >= 0) {
 313 +                                               read_gen_name(f_in_name,
 314 +                                                       fnamecmpbuf, NULL);
 315 +                                       }
 316 +                                       next_gen_i = read_int(batch_gen_fd);
 317 +                                       if (next_gen_i < 0)
 318 +                                               break;
 319 +                               }
 320                                 next_gen_i = -1;
 321                         }
 322
 323 @@ -411,6 +443,10 @@ int recv_files(int f_in, struct file_lis
 324
 325                 if (read_batch) {
 326                         while (i > next_gen_i) {
 327 +                               if (f_in_name >= 0 && next_gen_i >= 0) {
 328 +                                       read_gen_name(f_in_name, fnamecmpbuf,
 329 +                                                     NULL);
 330 +                               }
 331                                 next_gen_i = read_int(batch_gen_fd);
 332                                 if (next_gen_i == -1)
 333                                         next_gen_i = flist->count;
 334 @@ -421,8 +457,14 @@ int recv_files(int f_in, struct file_lis
 335                                 discard_receive_data(f_in, file->length);
 336                                 continue;
 337                         }
 338 +                       next_gen_i = -1;
 339                 }
 340
 341 +               if (f_in_name >= 0)
 342 +                       fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname);
 343 +               else
 344 +                       fnamecmp = fname;
 345 +
 346                 if (server_exclude_list.head
 347                     && check_exclude(&server_exclude_list, fname,
 348                                      S_ISDIR(file->mode)) < 0) {
 349 @@ -435,13 +477,7 @@ int recv_files(int f_in, struct file_lis
 350                         continue;
 351                 }
 352
 353 -               if (partial_dir) {
 354 -                       if ((partialptr = partial_dir_fname(fname)) != NULL)
 355 -                               fnamecmp = partialptr;
 356 -                       else
 357 -                               fnamecmp = fname;
 358 -               } else
 359 -                       fnamecmp = partialptr = fname;
 360 +               partialptr = partial_dir ? partial_dir_fname(fname) : fname;
 361
 362                 /* open the file */
 363                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 364 @@ -451,14 +487,6 @@ int recv_files(int f_in, struct file_lis
 365                         fd1 = do_open(fnamecmp, O_RDONLY, 0);
 366                 }
 367
 368 -               if (fd1 == -1 && compare_dest != NULL) {
 369 -                       /* try the file at compare_dest instead */
 370 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 371 -                                compare_dest, fname);
 372 -                       fnamecmp = fnamecmpbuf;
 373 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 374 -               }
 375 -
 376                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 377                         rsyserr(FERROR, errno, "fstat %s failed",
 378                                 full_fname(fnamecmp));