inplace.diff

   1 Patch from Mark Curtis to implement the --inplace option.
   2 Improved by Wayne Davison.
   3
   4 Run these commands after applying the patch:
   5
   6     autoconf
   7     autoheader
   8     ./configure
   9     make proto
  10
  11 --- configure.in        30 Apr 2004 18:03:33 -0000      1.196
  12 +++ configure.in        2 Jul 2004 21:10:25 -0000
  13 @@ -442,7 +442,7 @@ dnl AC_FUNC_MEMCMP
  14  AC_FUNC_UTIME_NULL
  15  AC_FUNC_ALLOCA
  16  AC_CHECK_FUNCS(waitpid wait4 getcwd strdup strerror chown chmod mknod mkfifo \
  17 -    fchmod fstat strchr readlink link utime utimes strftime mtrace \
  18 +    fchmod fstat strchr readlink link utime utimes strftime mtrace ftruncate \
  19      memmove lchown vsnprintf snprintf asprintf setsid glob strpbrk \
  20      strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid)
  21
  22 --- match.c     21 May 2004 08:27:04 -0000      1.62
  23 +++ match.c     2 Jul 2004 21:20:10 -0000
  24 @@ -23,6 +23,7 @@ extern int verbose;
  25  extern int am_server;
  26  extern int do_progress;
  27  extern int checksum_seed;
  28 +extern int inplace;
  29
  30  typedef unsigned short tag;
  31
  32 @@ -52,9 +53,27 @@ static size_t *tag_table;
  33  #define gettag2(s1,s2) (((s1) + (s2)) & 0xFFFF)
  34  #define gettag(sum) gettag2((sum)&0xFFFF,(sum)>>16)
  35
  36 +static struct sum_buf *current_sums;
  37 +static int current_s2length;
  38 +
  39  static int compare_targets(struct target *t1,struct target *t2)
  40  {
  41 -       return (int)t1->t - (int)t2->t;
  42 +       struct sum_buf *s1, *s2;
  43 +       int diff;
  44 +
  45 +       if (t1->t != t2->t)
  46 +               return t1->t < t2->t ? -1 : 1;
  47 +
  48 +       s1 = &current_sums[t1->i];
  49 +       s2 = &current_sums[t2->i];
  50 +
  51 +       if (s1->sum1 != s2->sum1)
  52 +               return s1->sum1 < s2->sum1 ? -1 : 1;
  53 +
  54 +       if ((diff = memcmp(s1->sum2, s2->sum2, current_s2length)) != 0)
  55 +               return diff;
  56 +
  57 +       return s1->offset - s2->offset;
  58  }
  59
  60
  61 @@ -74,6 +93,8 @@ static void build_hash_table(struct sum_
  62                 targets[i].t = gettag(s->sums[i].sum1);
  63         }
  64
  65 +       current_sums = s->sums;
  66 +       current_s2length = s->s2length;
  67         qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets);
  68
  69         for (i = 0; i < TABLESIZE; i++)
  70 @@ -192,12 +213,24 @@ static void hash_search(int f,struct sum
  71                         unsigned int l;
  72                         size_t i = targets[j].i;
  73
  74 -                       if (sum != s->sums[i].sum1)
  75 +                       if (sum != s->sums[i].sum1) {
  76 +                               if (done_csum2)
  77 +                                       break;
  78                                 continue;
  79 +                       }
  80
  81                         /* also make sure the two blocks are the same length */
  82                         l = MIN((OFF_T)s->blength, len-offset);
  83 -                       if (l != s->sums[i].len)
  84 +                       if (l != s->sums[i].len) {
  85 +                               if (done_csum2)
  86 +                                       break;
  87 +                               continue;
  88 +                       }
  89 +
  90 +                       /* inplace: ensure chunk's offset is either >= our
  91 +                        * offset or that the data didn't move. */
  92 +                       if (inplace && s->sums[i].offset < offset
  93 +                           && s->sums[i].i >= 0)
  94                                 continue;
  95
  96                         if (verbose > 3)
  97 @@ -215,15 +248,34 @@ static void hash_search(int f,struct sum
  98                                 continue;
  99                         }
 100
 101 -                       /* we've found a match, but now check to see
 102 -                        * if last_i can hint at a better match */
 103 +                       /* We've found a match, but now check to see if last_i
 104 +                        * can hint at a better match.  If inplace is enabled,
 105 +                        * the best possible match is one with an identical
 106 +                        * offset, so we prefer that over a last_i+1 match. */
 107 +                       if (inplace && s->sums[i].offset == offset) {
 108 +                               s->sums[i].i = -1;
 109 +                               j = s->count;
 110 +                       } else if (i == last_i + 1)
 111 +                               j = s->count;
 112                         for (j++; j < s->count && targets[j].t == t; j++) {
 113                                 size_t i2 = targets[j].i;
 114 -                               if (i2 == last_i + 1) {
 115 +                               if (i2 == last_i + 1
 116 +                                   || (inplace && s->sums[i2].offset == offset)) {
 117                                         if (sum != s->sums[i2].sum1)
 118                                                 break;
 119                                         if (memcmp(sum2,s->sums[i2].sum2,s->s2length) != 0)
 120                                                 break;
 121 +                                       if (inplace) {
 122 +                                               if (s->sums[i2].offset < offset
 123 +                                                   && s->sums[i2].i >= 0)
 124 +                                                       continue;
 125 +                                               i = i2;
 126 +                                               if (s->sums[i2].offset == offset) {
 127 +                                                       s->sums[i2].i = -1;
 128 +                                                       break;
 129 +                                               }
 130 +                                               continue;
 131 +                                       }
 132                                         /* we've found an adjacent match - the RLL coder
 133                                          * will be happy */
 134                                         i = i2;
 135 --- options.c   20 Jun 2004 19:47:05 -0000      1.157
 136 +++ options.c   2 Jul 2004 21:20:10 -0000
 137 @@ -94,6 +94,7 @@ int ignore_errors = 0;
 138  int modify_window = 0;
 139  int blocking_io = -1;
 140  int checksum_seed = 0;
 141 +int inplace = 0;
 142  unsigned int block_size = 0;
 143
 144
 145 @@ -149,6 +150,7 @@ char *bind_address;
 146  static void print_rsync_version(enum logcode f)
 147  {
 148         char const *got_socketpair = "no ";
 149 +       char const *have_inplace = "no ";
 150         char const *hardlinks = "no ";
 151         char const *links = "no ";
 152         char const *ipv6 = "no ";
 153 @@ -158,6 +160,10 @@ static void print_rsync_version(enum log
 154         got_socketpair = "";
 155  #endif
 156
 157 +#if HAVE_FTRUNCATE
 158 +       have_inplace = "";
 159 +#endif
 160 +
 161  #if SUPPORT_HARD_LINKS
 162         hardlinks = "";
 163  #endif
 164 @@ -183,8 +189,8 @@ static void print_rsync_version(enum log
 165         /* Note that this field may not have type ino_t.  It depends
 166          * on the complicated interaction between largefile feature
 167          * macros. */
 168 -       rprintf(f, "              %sIPv6, %d-bit system inums, %d-bit internal inums\n",
 169 -               ipv6,
 170 +       rprintf(f, "              %sinplace, %sIPv6, %d-bit system inums, %d-bit internal inums\n",
 171 +               have_inplace, ipv6,
 172                 (int) (sizeof dumstat->st_ino * 8),
 173                 (int) (sizeof (uint64) * 8));
 174  #ifdef MAINTAINER_MODE
 175 @@ -234,6 +240,7 @@ void usage(enum logcode F)
 176    rprintf(F,"     --backup-dir            make backups into this directory\n");
 177    rprintf(F,"     --suffix=SUFFIX         backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
 178    rprintf(F," -u, --update                update only (don't overwrite newer files)\n");
 179 +  rprintf(F,"     --inplace               update the destination file inplace (see man page)\n");
 180    rprintf(F," -K, --keep-dirlinks         treat symlinked dir on receiver as dir\n");
 181    rprintf(F," -l, --links                 copy symlinks as symlinks\n");
 182    rprintf(F," -L, --copy-links            copy the referent of all symlinks\n");
 183 @@ -341,6 +348,7 @@ static struct poptOption long_options[]
 184    {"sparse",          'S', POPT_ARG_NONE,   &sparse_files, 0, 0, 0 },
 185    {"cvs-exclude",     'C', POPT_ARG_NONE,   &cvs_exclude, 0, 0, 0 },
 186    {"update",          'u', POPT_ARG_NONE,   &update_only, 0, 0, 0 },
 187 +  {"inplace",          0,  POPT_ARG_NONE,   &inplace, 0, 0, 0 },
 188    {"keep-dirlinks",   'K', POPT_ARG_NONE,   &keep_dirlinks, 0, 0, 0 },
 189    {"links",           'l', POPT_ARG_NONE,   &preserve_links, 0, 0, 0 },
 190    {"copy-links",      'L', POPT_ARG_NONE,   &copy_links, 0, 0, 0 },
 191 @@ -739,6 +747,18 @@ int parse_arguments(int *argc, const cha
 192                         bwlimit_writemax = 512;
 193         }
 194
 195 +       if (inplace) {
 196 +#if HAVE_FTRUNCATE
 197 +               if (keep_partial)
 198 +                       keep_partial = 0;
 199 +#else
 200 +               snprintf(err_buf, sizeof err_buf,
 201 +                        "inplace is not supported on this %s\n",
 202 +                        am_server ? "server" : "client");
 203 +               return 0;
 204 +#endif
 205 +       }
 206 +
 207         if (files_from) {
 208                 char *colon;
 209                 if (*argc != 2 && !(am_server && am_sender && *argc == 1)) {
 210 @@ -963,6 +983,9 @@ void server_options(char **args,int *arg
 211         if (opt_ignore_existing && am_sender)
 212                 args[ac++] = "--ignore-existing";
 213
 214 +       if (inplace)
 215 +               args[ac++] = "--inplace";
 216 +
 217         if (tmpdir) {
 218                 args[ac++] = "--temp-dir";
 219                 args[ac++] = tmpdir;
 220 --- receiver.c  2 Jul 2004 18:23:57 -0000       1.86
 221 +++ receiver.c  2 Jul 2004 21:20:10 -0000
 222 @@ -48,6 +48,7 @@ extern int ignore_errors;
 223  extern int orig_umask;
 224  extern int keep_partial;
 225  extern int checksum_seed;
 226 +extern int inplace;
 227
 228  static void delete_one(char *fn, int is_dir)
 229  {
 230 @@ -255,16 +256,30 @@ static int receive_data(int f_in,struct
 231                         sum_update(map,len);
 232                 }
 233
 234 -               if (fd != -1 && write_file(fd, map, len) != (int)len) {
 235 -                       rsyserr(FERROR, errno, "write failed on %s",
 236 -                               full_fname(fname));
 237 -                       exit_cleanup(RERR_FILEIO);
 238 +               if (!inplace || offset != offset2) {
 239 +                       if (fd != -1 && write_file(fd, map, len) != (int)len) {
 240 +                               rsyserr(FERROR, errno, "write failed on %s",
 241 +                                       full_fname(fname));
 242 +                               exit_cleanup(RERR_FILEIO);
 243 +                       }
 244 +               } else {
 245 +                       flush_write_file(fd);
 246 +                       if (do_lseek(fd,(OFF_T)len,SEEK_CUR) != offset+len) {
 247 +                               rprintf(FERROR, "lseek failed on %s: %s, %lli, %lli, %i\n",
 248 +                                       full_fname(fname), strerror(errno), do_lseek(fd,0,SEEK_CUR), (offset+len), i);
 249 +                               exit_cleanup(RERR_FILEIO);
 250 +                       }
 251                 }
 252                 offset += len;
 253         }
 254
 255         flush_write_file(fd);
 256
 257 +#ifdef HAVE_FTRUNCATE
 258 +       if (inplace)
 259 +               ftruncate(fd, offset);
 260 +#endif
 261 +
 262         if (do_progress)
 263                 end_progress(total_size);
 264
 265 @@ -414,44 +429,59 @@ int recv_files(int f_in,struct file_list
 266                 } else
 267                         mapbuf = NULL;
 268
 269 -               if (!get_tmpname(fnametmp,fname)) {
 270 -                       if (mapbuf)
 271 -                               unmap_file(mapbuf);
 272 -                       if (fd1 != -1)
 273 -                               close(fd1);
 274 -                       continue;
 275 -               }
 276 +               /* We now check to see if we are writing file "inplace" */
 277 +               if (inplace)  {
 278 +                       fd2 = do_open(fnamecmp, O_WRONLY|O_CREAT, 0);
 279 +                       if (fd2 == -1) {
 280 +                               rsyserr(FERROR, errno, "open %s failed",
 281 +                                       full_fname(fnamecmp));
 282 +                               receive_data(f_in,mapbuf,-1,NULL,file->length);
 283 +                               if (mapbuf)
 284 +                                       unmap_file(mapbuf);
 285 +                               if (fd1 != -1)
 286 +                                       close(fd1);
 287 +                               continue;
 288 +                       }
 289 +               } else {
 290 +                       if (!get_tmpname(fnametmp,fname)) {
 291 +                               if (mapbuf)
 292 +                                       unmap_file(mapbuf);
 293 +                               if (fd1 != -1)
 294 +                                       close(fd1);
 295 +                               continue;
 296 +                       }
 297
 298 -               strlcpy(template, fnametmp, sizeof template);
 299 +                       strlcpy(template, fnametmp, sizeof template);
 300
 301 -               /* we initially set the perms without the
 302 -                * setuid/setgid bits to ensure that there is no race
 303 -                * condition. They are then correctly updated after
 304 -                * the lchown. Thanks to snabb@epipe.fi for pointing
 305 -                * this out.  We also set it initially without group
 306 -                * access because of a similar race condition. */
 307 -               fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
 308 -
 309 -               /* in most cases parent directories will already exist
 310 -                * because their information should have been previously
 311 -                * transferred, but that may not be the case with -R */
 312 -               if (fd2 == -1 && relative_paths && errno == ENOENT &&
 313 -                   create_directory_path(fnametmp, orig_umask) == 0) {
 314 -                       strlcpy(fnametmp, template, sizeof fnametmp);
 315 +                       /* we initially set the perms without the
 316 +                        * setuid/setgid bits to ensure that there is no race
 317 +                        * condition. They are then correctly updated after
 318 +                        * the lchown. Thanks to snabb@epipe.fi for pointing
 319 +                        * this out.  We also set it initially without group
 320 +                        * access because of a similar race condition. */
 321                         fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
 322 -               }
 323 -               if (fd2 == -1) {
 324 -                       rsyserr(FERROR, errno, "mkstemp %s failed",
 325 -                               full_fname(fnametmp));
 326 -                       receive_data(f_in,mapbuf,-1,NULL,file->length);
 327 -                       if (mapbuf)
 328 -                               unmap_file(mapbuf);
 329 -                       if (fd1 != -1)
 330 -                               close(fd1);
 331 -                       continue;
 332 -               }
 333
 334 -               cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2);
 335 +                       /* in most cases parent directories will already exist
 336 +                        * because their information should have been previously
 337 +                        * transferred, but that may not be the case with -R */
 338 +                       if (fd2 == -1 && relative_paths && errno == ENOENT
 339 +                           && create_directory_path(fnametmp, orig_umask) == 0) {
 340 +                               strlcpy(fnametmp, template, sizeof fnametmp);
 341 +                               fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
 342 +                       }
 343 +                       if (fd2 == -1) {
 344 +                               rsyserr(FERROR, errno, "mkstemp %s failed",
 345 +                                       full_fname(fnametmp));
 346 +                               receive_data(f_in,mapbuf,-1,NULL,file->length);
 347 +                               if (mapbuf)
 348 +                                       unmap_file(mapbuf);
 349 +                               if (fd1 != -1)
 350 +                                       close(fd1);
 351 +                               continue;
 352 +                       }
 353 +
 354 +                       cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2);
 355 +               }
 356
 357                 if (!am_server && verbose)
 358                         rprintf(FINFO, "%s\n", fname);
 359 --- rsync.c     2 Jul 2004 18:13:53 -0000       1.142
 360 +++ rsync.c     2 Jul 2004 21:20:11 -0000
 361 @@ -34,6 +34,7 @@ extern int force_delete;
 362  extern int recurse;
 363  extern int make_backups;
 364  extern char *backup_dir;
 365 +extern int inplace;
 366
 367
 368  /*
 369 @@ -239,6 +240,13 @@ void finish_transfer(char *fname, char *
 370         if (make_backups && !make_backup(fname))
 371                 return;
 372
 373 +       if (inplace) {
 374 +               if (verbose > 2)
 375 +                       rprintf(FINFO, "finishing %s\n", fname);
 376 +               set_perms(fname, file, NULL, 0);
 377 +               return;
 378 +       }
 379 +
 380         /* move tmp file over real file */
 381         if (verbose > 2)
 382                 rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
 383 --- rsync.yo    5 Jun 2004 16:16:30 -0000       1.171
 384 +++ rsync.yo    2 Jul 2004 21:20:12 -0000
 385 @@ -289,6 +289,7 @@ verb(
 386       --backup-dir            make backups into this directory
 387       --suffix=SUFFIX         backup suffix (default ~ w/o --backup-dir)
 388   -u, --update                update only (don't overwrite newer files)
 389 +     --inplace               update the destination file inplace
 390   -K, --keep-dirlinks         treat symlinked dir on receiver as dir
 391   -l, --links                 copy symlinks as symlinks
 392   -L, --copy-links            copy the referent of all symlinks
 393 @@ -484,6 +485,17 @@ dit(bf(-K, --keep-dirlinks)) On the rece
 394  pointing to a directory, it will be treated as matching a directory
 395  from the sender.
 396
 397 +dit(bf(--inplace)) This causes rsync not to create a new copy of the file
 398 +and then move it into place.  Instead rsync will overwrite the existing
 399 +file, meaning that the rsync algorithm can't extract the full ammount of
 400 +network reduction it might otherwise.
 401 +
 402 +This option is useful for transfer of large files with block based changes
 403 +and also on systems that are disk bound not network bound.
 404 +
 405 +WARNING: If the transfer is interrupted, you will have an inconsistent file
 406 +and the transfer should be run again.
 407 +
 408  dit(bf(-l, --links)) When symlinks are encountered, recreate the
 409  symlink on the destination.
 410