Modified to work with the trunk's hash_search() optimization.

[rsync/rsync-patches.git] / inplace.diff

Patch from Mark Curtis to implement the --inplace option.
Improved by Wayne Davison.

Run these commands after applying the patch:

    autoconf
    autoheader
    ./configure
    make

Optional:

    make rsync.1

--- configure.in	30 Apr 2004 18:03:33 -0000	1.196
+++ configure.in	3 Jul 2004 00:16:50 -0000
@@ -442,7 +442,7 @@ dnl AC_FUNC_MEMCMP
 AC_FUNC_UTIME_NULL
 AC_FUNC_ALLOCA
 AC_CHECK_FUNCS(waitpid wait4 getcwd strdup strerror chown chmod mknod mkfifo \
-    fchmod fstat strchr readlink link utime utimes strftime mtrace \
+    fchmod fstat strchr readlink link utime utimes strftime mtrace ftruncate \
     memmove lchown vsnprintf snprintf asprintf setsid glob strpbrk \
     strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid)
 
--- match.c	2 Jul 2004 23:35:30 -0000	1.63
+++ match.c	3 Jul 2004 00:16:50 -0000
@@ -23,6 +23,7 @@ extern int verbose;
 extern int am_server;
 extern int do_progress;
 extern int checksum_seed;
+extern int inplace;
 
 typedef unsigned short tag;
 
@@ -52,9 +53,23 @@ static size_t *tag_table;
 #define gettag2(s1,s2) (((s1) + (s2)) & 0xFFFF)
 #define gettag(sum) gettag2((sum)&0xFFFF,(sum)>>16)
 
+static struct sum_buf *current_sums;
+static int current_s2length;
+
 static int compare_targets(struct target *t1,struct target *t2)
 {
-	return (int)t1->t - (int)t2->t;
+	struct sum_buf *s1, *s2;
+
+	if (t1->t != t2->t)
+		return t1->t < t2->t ? -1 : 1;
+
+	s1 = &current_sums[t1->i];
+	s2 = &current_sums[t2->i];
+
+	if (s1->sum1 != s2->sum1)
+		return s1->sum1 < s2->sum1 ? -1 : 1;
+
+	return memcmp(s1->sum2, s2->sum2, current_s2length);
 }
 
 
@@ -74,6 +89,8 @@ static void build_hash_table(struct sum_
 		targets[i].t = gettag(s->sums[i].sum1);
 	}
 
+	current_sums = s->sums;
+	current_s2length = s->s2length;
 	qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets);
 
 	for (i = 0; i < TABLESIZE; i++)
@@ -192,12 +209,24 @@ static void hash_search(int f,struct sum
 			unsigned int l;
 			size_t i = targets[j].i;
 
-			if (sum != s->sums[i].sum1)
+			if (sum != s->sums[i].sum1) {
+				if (done_csum2)
+					break;
 				continue;
+			}
 
 			/* also make sure the two blocks are the same length */
 			l = MIN((OFF_T)s->blength, len-offset);
-			if (l != s->sums[i].len)
+			if (l != s->sums[i].len) {
+				if (done_csum2)
+					break;
+				continue;
+			}
+
+			/* inplace: ensure chunk's offset is either >= our
+			 * offset or that the data didn't move. */
+			if (inplace && s->sums[i].offset < offset
+			    && s->sums[i].i >= 0)
 				continue;
 
 			if (verbose > 3)
@@ -215,15 +244,39 @@ static void hash_search(int f,struct sum
 				continue;
 			}
 
-			/* we've found a match, but now check to see
-			 * if last_i can hint at a better match */
+			/* We've found a match, but now check to see if last_i
+			 * can hint at a better match.  If inplace is enabled,
+			 * the best possible match is one with an identical
+			 * offset, so we prefer that over a last_i+1 match. */
+			if (inplace) {
+				for (; j < s->count && targets[j].t == t; j++) {
+					size_t i2 = targets[j].i;
+					if (s->sums[i2].offset != offset)
+						continue;
+					if (i2 != i) {
+						if (sum != s->sums[i2].sum1)
+							break;
+						if (memcmp(sum2, s->sums[i2].sum2,
+							   s->s2length) != 0)
+							break;
+						i = i2;
+					}
+					/* Use this as a flag to indicate that
+					 * this chunk was at the same offset on
+					 * both the sender and the receiver. */
+					s->sums[i].i = -1;
+					goto set_last_i;
+				}
+			}
 			if (i != last_i + 1 && last_i + 1 < s->count
+			    && (!inplace || s->sums[last_i+1].offset >= offset || s->sums[last_i+1].i < 0)
 			    && sum == s->sums[last_i+1].sum1
 			    && memcmp(sum2, s->sums[last_i+1].sum2, s->s2length) == 0) {
 				/* we've found an adjacent match - the RLL coder
 				 * will be happy */
 				i = last_i + 1;
 			}
+		    set_last_i:
 			last_i = i;
 
 			matched(f,s,buf,offset,i);
--- options.c	20 Jun 2004 19:47:05 -0000	1.157
+++ options.c	3 Jul 2004 00:16:50 -0000
@@ -94,6 +94,7 @@ int ignore_errors = 0;
 int modify_window = 0;
 int blocking_io = -1;
 int checksum_seed = 0;
+int inplace = 0;
 unsigned int block_size = 0;
 
 
@@ -149,6 +150,7 @@ char *bind_address;
 static void print_rsync_version(enum logcode f)
 {
 	char const *got_socketpair = "no ";
+	char const *have_inplace = "no ";
 	char const *hardlinks = "no ";
 	char const *links = "no ";
 	char const *ipv6 = "no ";
@@ -158,6 +160,10 @@ static void print_rsync_version(enum log
 	got_socketpair = "";
 #endif
 
+#if HAVE_FTRUNCATE
+	have_inplace = "";
+#endif
+
 #if SUPPORT_HARD_LINKS
 	hardlinks = "";
 #endif
@@ -183,8 +189,8 @@ static void print_rsync_version(enum log
 	/* Note that this field may not have type ino_t.  It depends
 	 * on the complicated interaction between largefile feature
 	 * macros. */
-	rprintf(f, "              %sIPv6, %d-bit system inums, %d-bit internal inums\n",
-		ipv6,
+	rprintf(f, "              %sinplace, %sIPv6, %d-bit system inums, %d-bit internal inums\n",
+		have_inplace, ipv6,
 		(int) (sizeof dumstat->st_ino * 8),
 		(int) (sizeof (uint64) * 8));
 #ifdef MAINTAINER_MODE
@@ -234,6 +240,7 @@ void usage(enum logcode F)
   rprintf(F,"     --backup-dir            make backups into this directory\n");
   rprintf(F,"     --suffix=SUFFIX         backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
   rprintf(F," -u, --update                update only (don't overwrite newer files)\n");
+  rprintf(F,"     --inplace               update the destination file inplace (see man page)\n");
   rprintf(F," -K, --keep-dirlinks         treat symlinked dir on receiver as dir\n");
   rprintf(F," -l, --links                 copy symlinks as symlinks\n");
   rprintf(F," -L, --copy-links            copy the referent of all symlinks\n");
@@ -341,6 +348,7 @@ static struct poptOption long_options[] 
   {"sparse",          'S', POPT_ARG_NONE,   &sparse_files, 0, 0, 0 },
   {"cvs-exclude",     'C', POPT_ARG_NONE,   &cvs_exclude, 0, 0, 0 },
   {"update",          'u', POPT_ARG_NONE,   &update_only, 0, 0, 0 },
+  {"inplace",          0,  POPT_ARG_NONE,   &inplace, 0, 0, 0 },
   {"keep-dirlinks",   'K', POPT_ARG_NONE,   &keep_dirlinks, 0, 0, 0 },
   {"links",           'l', POPT_ARG_NONE,   &preserve_links, 0, 0, 0 },
   {"copy-links",      'L', POPT_ARG_NONE,   &copy_links, 0, 0, 0 },
@@ -739,6 +747,18 @@ int parse_arguments(int *argc, const cha
 			bwlimit_writemax = 512;
 	}
 
+	if (inplace) {
+#if HAVE_FTRUNCATE
+		if (keep_partial)
+			keep_partial = 0;
+#else
+		snprintf(err_buf, sizeof err_buf,
+			 "inplace is not supported on this %s\n",
+			 am_server ? "server" : "client");
+		return 0;
+#endif
+	}
+
 	if (files_from) {
 		char *colon;
 		if (*argc != 2 && !(am_server && am_sender && *argc == 1)) {
@@ -963,6 +983,9 @@ void server_options(char **args,int *arg
 	if (opt_ignore_existing && am_sender)
 		args[ac++] = "--ignore-existing";
 
+	if (inplace)
+		args[ac++] = "--inplace";
+
 	if (tmpdir) {
 		args[ac++] = "--temp-dir";
 		args[ac++] = tmpdir;
--- receiver.c	2 Jul 2004 18:23:57 -0000	1.86
+++ receiver.c	3 Jul 2004 00:16:50 -0000
@@ -48,6 +48,7 @@ extern int ignore_errors;
 extern int orig_umask;
 extern int keep_partial;
 extern int checksum_seed;
+extern int inplace;
 
 static void delete_one(char *fn, int is_dir)
 {
@@ -255,16 +256,30 @@ static int receive_data(int f_in,struct 
 			sum_update(map,len);
 		}
 
-		if (fd != -1 && write_file(fd, map, len) != (int)len) {
-			rsyserr(FERROR, errno, "write failed on %s",
-				full_fname(fname));
-			exit_cleanup(RERR_FILEIO);
+		if (!inplace || offset != offset2) {
+			if (fd != -1 && write_file(fd, map, len) != (int)len) {
+				rsyserr(FERROR, errno, "write failed on %s",
+					full_fname(fname));
+				exit_cleanup(RERR_FILEIO);
+			}
+		} else {
+			flush_write_file(fd);
+			if (do_lseek(fd,(OFF_T)len,SEEK_CUR) != offset+len) {
+				rprintf(FERROR, "lseek failed on %s: %s, %lli, %lli, %i\n",
+					full_fname(fname), strerror(errno), do_lseek(fd,0,SEEK_CUR), (offset+len), i);
+				exit_cleanup(RERR_FILEIO);
+			}
 		}
 		offset += len;
 	}
 
 	flush_write_file(fd);
 
+#ifdef HAVE_FTRUNCATE
+	if (inplace)
+		ftruncate(fd, offset);
+#endif
+
 	if (do_progress)
 		end_progress(total_size);
 
@@ -414,44 +429,59 @@ int recv_files(int f_in,struct file_list
 		} else
 			mapbuf = NULL;
 
-		if (!get_tmpname(fnametmp,fname)) {
-			if (mapbuf)
-				unmap_file(mapbuf);
-			if (fd1 != -1)
-				close(fd1);
-			continue;
-		}
+		/* We now check to see if we are writing file "inplace" */
+		if (inplace)  {
+			fd2 = do_open(fnamecmp, O_WRONLY|O_CREAT, 0);
+			if (fd2 == -1) {
+				rsyserr(FERROR, errno, "open %s failed",
+					full_fname(fnamecmp));
+				receive_data(f_in,mapbuf,-1,NULL,file->length);
+				if (mapbuf)
+					unmap_file(mapbuf);
+				if (fd1 != -1)
+					close(fd1);
+				continue;
+			}
+		} else {
+			if (!get_tmpname(fnametmp,fname)) {
+				if (mapbuf)
+					unmap_file(mapbuf);
+				if (fd1 != -1)
+					close(fd1);
+				continue;
+			}
 
-		strlcpy(template, fnametmp, sizeof template);
+			strlcpy(template, fnametmp, sizeof template);
 
-		/* we initially set the perms without the
-		 * setuid/setgid bits to ensure that there is no race
-		 * condition. They are then correctly updated after
-		 * the lchown. Thanks to snabb@epipe.fi for pointing
-		 * this out.  We also set it initially without group
-		 * access because of a similar race condition. */
-		fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
-
-		/* in most cases parent directories will already exist
-		 * because their information should have been previously
-		 * transferred, but that may not be the case with -R */
-		if (fd2 == -1 && relative_paths && errno == ENOENT &&
-		    create_directory_path(fnametmp, orig_umask) == 0) {
-			strlcpy(fnametmp, template, sizeof fnametmp);
+			/* we initially set the perms without the
+			 * setuid/setgid bits to ensure that there is no race
+			 * condition. They are then correctly updated after
+			 * the lchown. Thanks to snabb@epipe.fi for pointing
+			 * this out.  We also set it initially without group
+			 * access because of a similar race condition. */
 			fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
-		}
-		if (fd2 == -1) {
-			rsyserr(FERROR, errno, "mkstemp %s failed",
-				full_fname(fnametmp));
-			receive_data(f_in,mapbuf,-1,NULL,file->length);
-			if (mapbuf)
-				unmap_file(mapbuf);
-			if (fd1 != -1)
-				close(fd1);
-			continue;
-		}
 
-		cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2);
+			/* in most cases parent directories will already exist
+			 * because their information should have been previously
+			 * transferred, but that may not be the case with -R */
+			if (fd2 == -1 && relative_paths && errno == ENOENT
+			    && create_directory_path(fnametmp, orig_umask) == 0) {
+				strlcpy(fnametmp, template, sizeof fnametmp);
+				fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
+			}
+			if (fd2 == -1) {
+				rsyserr(FERROR, errno, "mkstemp %s failed",
+					full_fname(fnametmp));
+				receive_data(f_in,mapbuf,-1,NULL,file->length);
+				if (mapbuf)
+					unmap_file(mapbuf);
+				if (fd1 != -1)
+					close(fd1);
+				continue;
+			}
+
+			cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2);
+		}
 
 		if (!am_server && verbose)
 			rprintf(FINFO, "%s\n", fname);
--- rsync.c	2 Jul 2004 18:13:53 -0000	1.142
+++ rsync.c	3 Jul 2004 00:16:51 -0000
@@ -34,6 +34,7 @@ extern int force_delete;
 extern int recurse;
 extern int make_backups;
 extern char *backup_dir;
+extern int inplace;
 
 
 /*
@@ -239,6 +240,13 @@ void finish_transfer(char *fname, char *
 	if (make_backups && !make_backup(fname))
 		return;
 
+	if (inplace) {
+		if (verbose > 2)
+			rprintf(FINFO, "finishing %s\n", fname);
+		set_perms(fname, file, NULL, 0);
+		return;
+	}
+
 	/* move tmp file over real file */
 	if (verbose > 2)
 		rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
--- rsync.yo	5 Jun 2004 16:16:30 -0000	1.171
+++ rsync.yo	3 Jul 2004 00:16:52 -0000
@@ -289,6 +289,7 @@ verb(
      --backup-dir            make backups into this directory
      --suffix=SUFFIX         backup suffix (default ~ w/o --backup-dir)
  -u, --update                update only (don't overwrite newer files)
+     --inplace               update the destination file inplace
  -K, --keep-dirlinks         treat symlinked dir on receiver as dir
  -l, --links                 copy symlinks as symlinks
  -L, --copy-links            copy the referent of all symlinks
@@ -484,6 +485,17 @@ dit(bf(-K, --keep-dirlinks)) On the rece
 pointing to a directory, it will be treated as matching a directory
 from the sender.
 
+dit(bf(--inplace)) This causes rsync not to create a new copy of the file
+and then move it into place.  Instead rsync will overwrite the existing
+file, meaning that the rsync algorithm can't extract the full ammount of
+network reduction it might otherwise.
+
+This option is useful for transfer of large files with block based changes
+and also on systems that are disk bound not network bound.
+
+WARNING: If the transfer is interrupted, you will have an inconsistent file
+and the transfer should be run again.
+
 dit(bf(-l, --links)) When symlinks are encountered, recreate the
 symlink on the destination.