Patch from Mark Curtis to implement the --inplace option. Improved by Wayne Davison. Run these commands after applying the patch: autoconf autoheader ./configure make Optional: make rsync.1 --- configure.in 30 Apr 2004 18:03:33 -0000 1.196 +++ configure.in 3 Jul 2004 00:16:50 -0000 @@ -442,7 +442,7 @@ dnl AC_FUNC_MEMCMP AC_FUNC_UTIME_NULL AC_FUNC_ALLOCA AC_CHECK_FUNCS(waitpid wait4 getcwd strdup strerror chown chmod mknod mkfifo \ - fchmod fstat strchr readlink link utime utimes strftime mtrace \ + fchmod fstat strchr readlink link utime utimes strftime mtrace ftruncate \ memmove lchown vsnprintf snprintf asprintf setsid glob strpbrk \ strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid) --- match.c 2 Jul 2004 23:35:30 -0000 1.63 +++ match.c 3 Jul 2004 00:16:50 -0000 @@ -23,6 +23,7 @@ extern int verbose; extern int am_server; extern int do_progress; extern int checksum_seed; +extern int inplace; typedef unsigned short tag; @@ -52,9 +53,23 @@ static size_t *tag_table; #define gettag2(s1,s2) (((s1) + (s2)) & 0xFFFF) #define gettag(sum) gettag2((sum)&0xFFFF,(sum)>>16) +static struct sum_buf *current_sums; +static int current_s2length; + static int compare_targets(struct target *t1,struct target *t2) { - return (int)t1->t - (int)t2->t; + struct sum_buf *s1, *s2; + + if (t1->t != t2->t) + return t1->t < t2->t ? -1 : 1; + + s1 = ¤t_sums[t1->i]; + s2 = ¤t_sums[t2->i]; + + if (s1->sum1 != s2->sum1) + return s1->sum1 < s2->sum1 ? -1 : 1; + + return memcmp(s1->sum2, s2->sum2, current_s2length); } @@ -74,6 +89,8 @@ static void build_hash_table(struct sum_ targets[i].t = gettag(s->sums[i].sum1); } + current_sums = s->sums; + current_s2length = s->s2length; qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets); for (i = 0; i < TABLESIZE; i++) @@ -192,12 +209,24 @@ static void hash_search(int f,struct sum unsigned int l; size_t i = targets[j].i; - if (sum != s->sums[i].sum1) + if (sum != s->sums[i].sum1) { + if (done_csum2) + break; continue; + } /* also make sure the two blocks are the same length */ l = MIN((OFF_T)s->blength, len-offset); - if (l != s->sums[i].len) + if (l != s->sums[i].len) { + if (done_csum2) + break; + continue; + } + + /* inplace: ensure chunk's offset is either >= our + * offset or that the data didn't move. */ + if (inplace && s->sums[i].offset < offset + && s->sums[i].i >= 0) continue; if (verbose > 3) @@ -215,15 +244,39 @@ static void hash_search(int f,struct sum continue; } - /* we've found a match, but now check to see - * if last_i can hint at a better match */ + /* We've found a match, but now check to see if last_i + * can hint at a better match. If inplace is enabled, + * the best possible match is one with an identical + * offset, so we prefer that over a last_i+1 match. */ + if (inplace) { + for (; j < s->count && targets[j].t == t; j++) { + size_t i2 = targets[j].i; + if (s->sums[i2].offset != offset) + continue; + if (i2 != i) { + if (sum != s->sums[i2].sum1) + break; + if (memcmp(sum2, s->sums[i2].sum2, + s->s2length) != 0) + break; + i = i2; + } + /* Use this as a flag to indicate that + * this chunk was at the same offset on + * both the sender and the receiver. */ + s->sums[i].i = -1; + goto set_last_i; + } + } if (i != last_i + 1 && last_i + 1 < s->count + && (!inplace || s->sums[last_i+1].offset >= offset || s->sums[last_i+1].i < 0) && sum == s->sums[last_i+1].sum1 && memcmp(sum2, s->sums[last_i+1].sum2, s->s2length) == 0) { /* we've found an adjacent match - the RLL coder * will be happy */ i = last_i + 1; } + set_last_i: last_i = i; matched(f,s,buf,offset,i); --- options.c 20 Jun 2004 19:47:05 -0000 1.157 +++ options.c 3 Jul 2004 00:16:50 -0000 @@ -94,6 +94,7 @@ int ignore_errors = 0; int modify_window = 0; int blocking_io = -1; int checksum_seed = 0; +int inplace = 0; unsigned int block_size = 0; @@ -149,6 +150,7 @@ char *bind_address; static void print_rsync_version(enum logcode f) { char const *got_socketpair = "no "; + char const *have_inplace = "no "; char const *hardlinks = "no "; char const *links = "no "; char const *ipv6 = "no "; @@ -158,6 +160,10 @@ static void print_rsync_version(enum log got_socketpair = ""; #endif +#if HAVE_FTRUNCATE + have_inplace = ""; +#endif + #if SUPPORT_HARD_LINKS hardlinks = ""; #endif @@ -183,8 +189,8 @@ static void print_rsync_version(enum log /* Note that this field may not have type ino_t. It depends * on the complicated interaction between largefile feature * macros. */ - rprintf(f, " %sIPv6, %d-bit system inums, %d-bit internal inums\n", - ipv6, + rprintf(f, " %sinplace, %sIPv6, %d-bit system inums, %d-bit internal inums\n", + have_inplace, ipv6, (int) (sizeof dumstat->st_ino * 8), (int) (sizeof (uint64) * 8)); #ifdef MAINTAINER_MODE @@ -234,6 +240,7 @@ void usage(enum logcode F) rprintf(F," --backup-dir make backups into this directory\n"); rprintf(F," --suffix=SUFFIX backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX); rprintf(F," -u, --update update only (don't overwrite newer files)\n"); + rprintf(F," --inplace update the destination file inplace (see man page)\n"); rprintf(F," -K, --keep-dirlinks treat symlinked dir on receiver as dir\n"); rprintf(F," -l, --links copy symlinks as symlinks\n"); rprintf(F," -L, --copy-links copy the referent of all symlinks\n"); @@ -341,6 +348,7 @@ static struct poptOption long_options[] {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, {"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 }, {"update", 'u', POPT_ARG_NONE, &update_only, 0, 0, 0 }, + {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, {"keep-dirlinks", 'K', POPT_ARG_NONE, &keep_dirlinks, 0, 0, 0 }, {"links", 'l', POPT_ARG_NONE, &preserve_links, 0, 0, 0 }, {"copy-links", 'L', POPT_ARG_NONE, ©_links, 0, 0, 0 }, @@ -739,6 +747,18 @@ int parse_arguments(int *argc, const cha bwlimit_writemax = 512; } + if (inplace) { +#if HAVE_FTRUNCATE + if (keep_partial) + keep_partial = 0; +#else + snprintf(err_buf, sizeof err_buf, + "inplace is not supported on this %s\n", + am_server ? "server" : "client"); + return 0; +#endif + } + if (files_from) { char *colon; if (*argc != 2 && !(am_server && am_sender && *argc == 1)) { @@ -963,6 +983,9 @@ void server_options(char **args,int *arg if (opt_ignore_existing && am_sender) args[ac++] = "--ignore-existing"; + if (inplace) + args[ac++] = "--inplace"; + if (tmpdir) { args[ac++] = "--temp-dir"; args[ac++] = tmpdir; --- receiver.c 2 Jul 2004 18:23:57 -0000 1.86 +++ receiver.c 3 Jul 2004 00:16:50 -0000 @@ -48,6 +48,7 @@ extern int ignore_errors; extern int orig_umask; extern int keep_partial; extern int checksum_seed; +extern int inplace; static void delete_one(char *fn, int is_dir) { @@ -255,16 +256,30 @@ static int receive_data(int f_in,struct sum_update(map,len); } - if (fd != -1 && write_file(fd, map, len) != (int)len) { - rsyserr(FERROR, errno, "write failed on %s", - full_fname(fname)); - exit_cleanup(RERR_FILEIO); + if (!inplace || offset != offset2) { + if (fd != -1 && write_file(fd, map, len) != (int)len) { + rsyserr(FERROR, errno, "write failed on %s", + full_fname(fname)); + exit_cleanup(RERR_FILEIO); + } + } else { + flush_write_file(fd); + if (do_lseek(fd,(OFF_T)len,SEEK_CUR) != offset+len) { + rprintf(FERROR, "lseek failed on %s: %s, %lli, %lli, %i\n", + full_fname(fname), strerror(errno), do_lseek(fd,0,SEEK_CUR), (offset+len), i); + exit_cleanup(RERR_FILEIO); + } } offset += len; } flush_write_file(fd); +#ifdef HAVE_FTRUNCATE + if (inplace) + ftruncate(fd, offset); +#endif + if (do_progress) end_progress(total_size); @@ -414,44 +429,59 @@ int recv_files(int f_in,struct file_list } else mapbuf = NULL; - if (!get_tmpname(fnametmp,fname)) { - if (mapbuf) - unmap_file(mapbuf); - if (fd1 != -1) - close(fd1); - continue; - } + /* We now check to see if we are writing file "inplace" */ + if (inplace) { + fd2 = do_open(fnamecmp, O_WRONLY|O_CREAT, 0); + if (fd2 == -1) { + rsyserr(FERROR, errno, "open %s failed", + full_fname(fnamecmp)); + receive_data(f_in,mapbuf,-1,NULL,file->length); + if (mapbuf) + unmap_file(mapbuf); + if (fd1 != -1) + close(fd1); + continue; + } + } else { + if (!get_tmpname(fnametmp,fname)) { + if (mapbuf) + unmap_file(mapbuf); + if (fd1 != -1) + close(fd1); + continue; + } - strlcpy(template, fnametmp, sizeof template); + strlcpy(template, fnametmp, sizeof template); - /* we initially set the perms without the - * setuid/setgid bits to ensure that there is no race - * condition. They are then correctly updated after - * the lchown. Thanks to snabb@epipe.fi for pointing - * this out. We also set it initially without group - * access because of a similar race condition. */ - fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS); - - /* in most cases parent directories will already exist - * because their information should have been previously - * transferred, but that may not be the case with -R */ - if (fd2 == -1 && relative_paths && errno == ENOENT && - create_directory_path(fnametmp, orig_umask) == 0) { - strlcpy(fnametmp, template, sizeof fnametmp); + /* we initially set the perms without the + * setuid/setgid bits to ensure that there is no race + * condition. They are then correctly updated after + * the lchown. Thanks to snabb@epipe.fi for pointing + * this out. We also set it initially without group + * access because of a similar race condition. */ fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS); - } - if (fd2 == -1) { - rsyserr(FERROR, errno, "mkstemp %s failed", - full_fname(fnametmp)); - receive_data(f_in,mapbuf,-1,NULL,file->length); - if (mapbuf) - unmap_file(mapbuf); - if (fd1 != -1) - close(fd1); - continue; - } - cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2); + /* in most cases parent directories will already exist + * because their information should have been previously + * transferred, but that may not be the case with -R */ + if (fd2 == -1 && relative_paths && errno == ENOENT + && create_directory_path(fnametmp, orig_umask) == 0) { + strlcpy(fnametmp, template, sizeof fnametmp); + fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS); + } + if (fd2 == -1) { + rsyserr(FERROR, errno, "mkstemp %s failed", + full_fname(fnametmp)); + receive_data(f_in,mapbuf,-1,NULL,file->length); + if (mapbuf) + unmap_file(mapbuf); + if (fd1 != -1) + close(fd1); + continue; + } + + cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2); + } if (!am_server && verbose) rprintf(FINFO, "%s\n", fname); --- rsync.c 2 Jul 2004 18:13:53 -0000 1.142 +++ rsync.c 3 Jul 2004 00:16:51 -0000 @@ -34,6 +34,7 @@ extern int force_delete; extern int recurse; extern int make_backups; extern char *backup_dir; +extern int inplace; /* @@ -239,6 +240,13 @@ void finish_transfer(char *fname, char * if (make_backups && !make_backup(fname)) return; + if (inplace) { + if (verbose > 2) + rprintf(FINFO, "finishing %s\n", fname); + set_perms(fname, file, NULL, 0); + return; + } + /* move tmp file over real file */ if (verbose > 2) rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname); --- rsync.yo 5 Jun 2004 16:16:30 -0000 1.171 +++ rsync.yo 3 Jul 2004 00:16:52 -0000 @@ -289,6 +289,7 @@ verb( --backup-dir make backups into this directory --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) -u, --update update only (don't overwrite newer files) + --inplace update the destination file inplace -K, --keep-dirlinks treat symlinked dir on receiver as dir -l, --links copy symlinks as symlinks -L, --copy-links copy the referent of all symlinks @@ -484,6 +485,17 @@ dit(bf(-K, --keep-dirlinks)) On the rece pointing to a directory, it will be treated as matching a directory from the sender. +dit(bf(--inplace)) This causes rsync not to create a new copy of the file +and then move it into place. Instead rsync will overwrite the existing +file, meaning that the rsync algorithm can't extract the full ammount of +network reduction it might otherwise. + +This option is useful for transfer of large files with block based changes +and also on systems that are disk bound not network bound. + +WARNING: If the transfer is interrupted, you will have an inconsistent file +and the transfer should be run again. + dit(bf(-l, --links)) When symlinks are encountered, recreate the symlink on the destination.