g2r-basis-filename.diff

   1 Depends-On-Patch: partial-dir.diff
   2
   3 Added a pipe from the generator to the receiver that communicates
   4 what basis file we used to generate the file data (if it was not
   5 the default name).  This optimizes away the basis-file search in
   6 the receiver and makes future options that do more basis-file
   7 searching more efficient (such as the --fuzzy option and the
   8 support for multiple --compare-dest options).
   9
  10 You must run "make proto" before compiling.
  11
  12 --- orig/generator.c    2004-07-28 09:58:28
  13 +++ generator.c 2004-07-28 10:14:15
  14 @@ -259,9 +259,9 @@ static void generate_and_send_sums(int f
  15   * out.  It might be wrong.
  16   */
  17  static void recv_generator(char *fname, struct file_struct *file, int i,
  18 -                          int f_out)
  19 +                          int f_out, int f_out_name)
  20  {
  21 -       int fd;
  22 +       int fd = -1;
  23         STRUCT_STAT st;
  24         int statret, stat_errno;
  25         char *fnamecmp;
  26 @@ -437,13 +437,16 @@ static void recv_generator(char *fname,
  27                                                         safe_fname(fname));
  28                                         }
  29                                         fnamecmp = fnamecmpbuf;
  30 -                               }
  31 +                               } else
  32 +                                       *fnamecmpbuf = '\0';
  33                         } else
  34  #endif
  35                                 fnamecmp = fnamecmpbuf;
  36                         statret = 0;
  37 -               }
  38 -       }
  39 +               } else
  40 +                       *fnamecmpbuf = '\0';
  41 +       } else
  42 +               *fnamecmpbuf = '\0';
  43
  44         if (statret == 0 && !S_ISREG(st.st_mode)) {
  45                 if (delete_file(fname) != 0)
  46 @@ -455,11 +458,9 @@ static void recv_generator(char *fname,
  47         if (statret == -1) {
  48                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
  49                         return;
  50 -               if (stat_errno == ENOENT) {
  51 -                       write_int(f_out,i);
  52 -                       if (!dry_run && !read_batch)
  53 -                               write_sum_head(f_out, NULL);
  54 -               } else if (verbose > 1) {
  55 +               if (stat_errno == ENOENT)
  56 +                       goto notify_others;
  57 +               if (verbose > 1) {
  58                         rsyserr(FERROR, stat_errno,
  59                                 "recv_generator: failed to stat %s",
  60                                 full_fname(fname));
  61 @@ -467,13 +468,13 @@ static void recv_generator(char *fname,
  62                 return;
  63         }
  64
  65 -       if (opt_ignore_existing && fnamecmp == fname) {
  66 +       if (opt_ignore_existing && !*fnamecmpbuf) {
  67                 if (verbose > 1)
  68                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
  69                 return;
  70         }
  71
  72 -       if (update_only && fnamecmp == fname
  73 +       if (update_only && !*fnamecmpbuf
  74             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
  75                 if (verbose > 1)
  76                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
  77 @@ -481,21 +482,17 @@ static void recv_generator(char *fname,
  78         }
  79
  80         if (skip_file(fname, file, &st)) {
  81 -               if (fnamecmp == fname)
  82 +               if (!*fnamecmpbuf)
  83                         set_perms(fname, file, &st, PERMS_REPORT);
  84                 return;
  85         }
  86
  87 -       if (dry_run || read_batch) {
  88 -               write_int(f_out,i);
  89 -               return;
  90 -       }
  91 -
  92 -       if (whole_file > 0) {
  93 -               write_int(f_out,i);
  94 -               write_sum_head(f_out, NULL);
  95 -               return;
  96 +       if (dry_run || whole_file) {
  97 +               statret = -1;
  98 +               goto notify_others;
  99         }
 100 +       if (read_batch)
 101 +               goto notify_others;
 102
 103         if (partial_dir) {
 104                 STRUCT_STAT st2;
 105 @@ -516,9 +513,8 @@ static void recv_generator(char *fname,
 106                 /* pretend the file didn't exist */
 107                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 108                         return;
 109 -               write_int(f_out,i);
 110 -               write_sum_head(f_out, NULL);
 111 -               return;
 112 +               statret = -1;
 113 +               goto notify_others;
 114         }
 115
 116         if (verbose > 3) {
 117 @@ -529,14 +525,41 @@ static void recv_generator(char *fname,
 118         if (verbose > 2)
 119                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 120
 121 -       write_int(f_out,i);
 122 -       generate_and_send_sums(fd, st.st_size, f_out);
 123 +notify_others:
 124 +       if (f_out_name >= 0) {
 125 +               uchar lenbuf[3], *lb = lenbuf;
 126 +               int len = statret == -1 ? 0 : strlen(fnamecmpbuf);
 127 +               if (len > 0x7F) {
 128 +#if MAXPATHLEN > 0x7FFF
 129 +                       *lb++ = len / 0x10000 + 0x80;
 130 +                       *lb++ = len / 0x100;
 131 +#else
 132 +                       *lb++ = len / 0x100 + 0x80;
 133 +#endif
 134 +               }
 135 +               *lb = len;
 136 +               write_buf(f_out_name, lenbuf, lb - lenbuf + 1);
 137 +               if (len)
 138 +                       write_buf(f_out_name, fnamecmpbuf, len);
 139 +               io_flush(NORMAL_FLUSH); /* XXX make this more efficient! */
 140 +       }
 141 +
 142 +       write_int(f_out, i);
 143 +
 144 +       if (dry_run || read_batch)
 145 +               return;
 146
 147 -       close(fd);
 148 +       if (statret == 0) {
 149 +               generate_and_send_sums(fd, st.st_size, f_out);
 150 +
 151 +               close(fd);
 152 +       } else
 153 +               write_sum_head(f_out, NULL);
 154  }
 155
 156
 157 -void generate_files(int f_out, struct file_list *flist, char *local_name)
 158 +void generate_files(int f_out, struct file_list *flist, char *local_name,
 159 +                   int f_out_name)
 160  {
 161         int i;
 162         int phase = 0;
 163 @@ -577,7 +600,7 @@ void generate_files(int f_out, struct fi
 164                 }
 165
 166                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 167 -                              file, i, f_out);
 168 +                              file, i, f_out, f_out_name);
 169         }
 170
 171         phase++;
 172 @@ -594,7 +617,7 @@ void generate_files(int f_out, struct fi
 173         while ((i = get_redo_num()) != -1) {
 174                 struct file_struct *file = flist->files[i];
 175                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 176 -                              file, i, f_out);
 177 +                              file, i, f_out, f_out_name);
 178         }
 179
 180         phase++;
 181 @@ -613,7 +636,7 @@ void generate_files(int f_out, struct fi
 182                 if (!file->basename || !S_ISDIR(file->mode))
 183                         continue;
 184                 recv_generator(local_name ? local_name : f_name(file),
 185 -                              file, i, -1);
 186 +                              file, i, -1, -1);
 187         }
 188
 189         if (verbose > 2)
 190 --- orig/main.c 2004-07-24 16:52:09
 191 +++ main.c      2004-07-22 00:10:43
 192 @@ -58,6 +58,7 @@ extern int filesfrom_fd;
 193  extern pid_t cleanup_child_pid;
 194  extern char *files_from;
 195  extern char *remote_filesfrom_file;
 196 +extern char *compare_dest;
 197  extern char *rsync_path;
 198  extern char *shell_cmd;
 199  extern char *batch_name;
 200 @@ -456,7 +457,8 @@ static int do_recv(int f_in,int f_out,st
 201  {
 202         int pid;
 203         int status = 0;
 204 -       int error_pipe[2];
 205 +       int error_pipe[2], name_pipe[2];
 206 +       BOOL need_name_pipe = compare_dest && !dry_run;
 207
 208         if (preserve_hard_links)
 209                 init_hard_links(flist);
 210 @@ -467,8 +469,9 @@ static int do_recv(int f_in,int f_out,st
 211                         delete_files(flist);
 212         }
 213
 214 -       if (fd_pair(error_pipe) < 0) {
 215 -               rprintf(FERROR,"error pipe failed in do_recv\n");
 216 +       if (fd_pair(error_pipe) < 0
 217 +           || (need_name_pipe && fd_pair(name_pipe) < 0)) {
 218 +               rprintf(FERROR, "fd_pair() failed in do_recv\n");
 219                 exit_cleanup(RERR_SOCKETIO);
 220         }
 221
 222 @@ -476,6 +479,11 @@ static int do_recv(int f_in,int f_out,st
 223
 224         if ((pid = do_fork()) == 0) {
 225                 close(error_pipe[0]);
 226 +               if (need_name_pipe) {
 227 +                       close(name_pipe[1]);
 228 +                       set_blocking(name_pipe[0]);
 229 +               } else
 230 +                       name_pipe[0] = -1;
 231                 if (f_in != f_out)
 232                         close(f_out);
 233
 234 @@ -485,7 +493,7 @@ static int do_recv(int f_in,int f_out,st
 235                 /* set place to send errors */
 236                 set_msg_fd_out(error_pipe[1]);
 237
 238 -               recv_files(f_in,flist,local_name);
 239 +               recv_files(f_in, flist, local_name, name_pipe[0]);
 240                 io_flush(FULL_FLUSH);
 241                 report(f_in);
 242
 243 @@ -503,6 +511,11 @@ static int do_recv(int f_in,int f_out,st
 244                 stop_write_batch();
 245
 246         close(error_pipe[1]);
 247 +       if (need_name_pipe) {
 248 +               close(name_pipe[0]);
 249 +               set_nonblocking(name_pipe[1]);
 250 +       } else
 251 +               name_pipe[1] = -1;
 252         if (f_in != f_out)
 253                 close(f_in);
 254
 255 @@ -510,7 +523,7 @@ static int do_recv(int f_in,int f_out,st
 256
 257         set_msg_fd_in(error_pipe[0]);
 258
 259 -       generate_files(f_out, flist, local_name);
 260 +       generate_files(f_out, flist, local_name, name_pipe[1]);
 261
 262         get_redo_num(); /* Read final MSG_DONE and any prior messages. */
 263         report(-1);
 264 --- orig/receiver.c     2004-07-27 23:26:20
 265 +++ receiver.c  2004-07-23 21:59:07
 266 @@ -325,6 +325,30 @@ static int receive_data(int f_in, char *
 267  }
 268
 269
 270 +static char *read_gen_name(int fd, char *buf, char *realname)
 271 +{
 272 +       int len = read_byte(fd);
 273 +       if (len & 0x80) {
 274 +#if MAXPATHLEN > 32767
 275 +               uchar lenbuf[2];
 276 +               read_buf(fd, (char *)lenbuf, 2);
 277 +               len = (len & ~0x80) * 0x10000 + lenbuf[0] * 0x100 + lenbuf[1];
 278 +#else
 279 +               len = (len & ~0x80) * 0x100 + read_byte(fd);
 280 +#endif
 281 +       }
 282 +       if (len) {
 283 +               if (len >= MAXPATHLEN) {
 284 +                       rprintf(FERROR, "bogus data on generator name pipe\n");
 285 +                       exit_cleanup(RERR_PROTOCOL);
 286 +               }
 287 +               read_sbuf(fd, buf, len);
 288 +               return buf;
 289 +       }
 290 +       return realname;
 291 +}
 292 +
 293 +
 294  static void discard_receive_data(int f_in, OFF_T length)
 295  {
 296         receive_data(f_in, NULL, -1, 0, NULL, -1, length);
 297 @@ -335,7 +359,8 @@ static void discard_receive_data(int f_i
 298   * main routine for receiver process.
 299   *
 300   * Receiver process runs on the same host as the generator process. */
 301 -int recv_files(int f_in, struct file_list *flist, char *local_name)
 302 +int recv_files(int f_in, struct file_list *flist, char *local_name,
 303 +              int f_in_name)
 304  {
 305         int next_gen_i = -1;
 306         int fd1,fd2;
 307 @@ -364,8 +389,15 @@ int recv_files(int f_in, struct file_lis
 308                 i = read_int(f_in);
 309                 if (i == -1) {
 310                         if (read_batch) {
 311 -                               if (next_gen_i != flist->count)
 312 -                                       while (read_int(batch_gen_fd) != -1) {}
 313 +                               while (next_gen_i < flist->count) {
 314 +                                       if (f_in_name >= 0 && next_gen_i >= 0) {
 315 +                                               read_gen_name(f_in_name,
 316 +                                                       fnamecmpbuf, NULL);
 317 +                                       }
 318 +                                       next_gen_i = read_int(batch_gen_fd);
 319 +                                       if (next_gen_i < 0)
 320 +                                               break;
 321 +                               }
 322                                 next_gen_i = -1;
 323                         }
 324
 325 @@ -413,6 +445,10 @@ int recv_files(int f_in, struct file_lis
 326
 327                 if (read_batch) {
 328                         while (i > next_gen_i) {
 329 +                               if (f_in_name >= 0 && next_gen_i >= 0) {
 330 +                                       read_gen_name(f_in_name, fnamecmpbuf,
 331 +                                                     NULL);
 332 +                               }
 333                                 next_gen_i = read_int(batch_gen_fd);
 334                                 if (next_gen_i == -1)
 335                                         next_gen_i = flist->count;
 336 @@ -423,8 +459,14 @@ int recv_files(int f_in, struct file_lis
 337                                 discard_receive_data(f_in, file->length);
 338                                 continue;
 339                         }
 340 +                       next_gen_i = -1;
 341                 }
 342
 343 +               if (f_in_name >= 0)
 344 +                       fnamecmp = read_gen_name(f_in_name, fnamecmpbuf, fname);
 345 +               else
 346 +                       fnamecmp = fname;
 347 +
 348                 if (server_exclude_list.head
 349                     && check_exclude(&server_exclude_list, fname,
 350                                      S_ISDIR(file->mode)) < 0) {
 351 @@ -437,13 +479,7 @@ int recv_files(int f_in, struct file_lis
 352                         continue;
 353                 }
 354
 355 -               if (partial_dir) {
 356 -                       if ((partialptr = partial_dir_fname(fname)) != NULL)
 357 -                               fnamecmp = partialptr;
 358 -                       else
 359 -                               fnamecmp = fname;
 360 -               } else
 361 -                       fnamecmp = partialptr = fname;
 362 +               partialptr = partial_dir ? partial_dir_fname(fname) : fname;
 363
 364                 /* open the file */
 365                 fd1 = do_open(fnamecmp, O_RDONLY, 0);
 366 @@ -453,14 +489,6 @@ int recv_files(int f_in, struct file_lis
 367                         fd1 = do_open(fnamecmp, O_RDONLY, 0);
 368                 }
 369
 370 -               if (fd1 == -1 && compare_dest != NULL) {
 371 -                       /* try the file at compare_dest instead */
 372 -                       pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
 373 -                                compare_dest, fname);
 374 -                       fnamecmp = fnamecmpbuf;
 375 -                       fd1 = do_open(fnamecmp, O_RDONLY, 0);
 376 -               }
 377 -
 378                 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
 379                         rsyserr(FERROR, errno, "fstat %s failed",
 380                                 full_fname(fnamecmp));