Fixed failing hunks.

[rsync/rsync-patches.git] / preallocate.diff

This patch adds the --preallocate option that asks rsync to preallocate the
copied files.  This slows down the copy, but should reduce fragmentation on
systems that need that.

To use this patch, run these commands for a successful build:

    patch -p1 <patches/preallocate.diff
    ./prepare-source
    ./configure
    make

--- old/configure.in
+++ new/configure.in
@@ -561,7 +561,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd
     strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
     setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
     strerror putenv iconv_open locale_charset nl_langinfo getxattr \
-    extattr_get_link sigaction sigprocmask)
+    extattr_get_link sigaction sigprocmask posix_fallocate)
 
 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
 if test $ac_cv_func_getpgrp = yes; then
--- old/options.c
+++ new/options.c
@@ -70,6 +70,7 @@ int remove_source_files = 0;
 int one_file_system = 0;
 int protocol_version = PROTOCOL_VERSION;
 int sparse_files = 0;
+int preallocate_files = 0;
 int do_compression = 0;
 int def_compress_level = Z_DEFAULT_COMPRESSION;
 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
@@ -212,6 +213,7 @@ static void print_rsync_version(enum log
 	char const *links = "no ";
 	char const *iconv = "no ";
 	char const *ipv6 = "no ";
+	char const *preallocation = "no ";
 	STRUCT_STAT *dumstat;
 
 #if SUBPROTOCOL_VERSION != 0
@@ -241,6 +243,9 @@ static void print_rsync_version(enum log
 #ifdef ICONV_OPTION
 	iconv = "";
 #endif
+#ifdef SUPPORT_PREALLOCATION
+	preallocation = "";
+#endif
 
 	rprintf(f, "%s  version %s  protocol version %d%s\n",
 		RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
@@ -254,8 +259,8 @@ static void print_rsync_version(enum log
 		(int)(sizeof (int64) * 8));
 	rprintf(f, "    %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
 		got_socketpair, hardlinks, links, ipv6, have_inplace);
-	rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv\n",
-		have_inplace, acls, xattrs, iconv);
+	rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv, %spreallocation\n",
+		have_inplace, acls, xattrs, iconv, preallocation);
 
 #ifdef MAINTAINER_MODE
 	rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
@@ -341,6 +346,9 @@ void usage(enum logcode F)
   rprintf(F,"     --fake-super            store/recover privileged attrs using xattrs\n");
 #endif
   rprintf(F," -S, --sparse                handle sparse files efficiently\n");
+#ifdef SUPPORT_PREALLOCATION
+  rprintf(F,"     --preallocate           posix_fallocate dest files before writing them\n");
+#endif
   rprintf(F," -n, --dry-run               show what would have been transferred\n");
   rprintf(F," -W, --whole-file            copy files whole (without rsync algorithm)\n");
   rprintf(F," -x, --one-file-system       don't cross filesystem boundaries\n");
@@ -512,6 +520,7 @@ static struct poptOption long_options[] 
   {"max-size",         0,  POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
   {"min-size",         0,  POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
   {"sparse",          'S', POPT_ARG_NONE,   &sparse_files, 0, 0, 0 },
+  {"preallocate",      0,  POPT_ARG_NONE,   &preallocate_files, 0, 0, 0},
   {"inplace",          0,  POPT_ARG_NONE,   &inplace, 0, 0, 0 },
   {"append",           0,  POPT_ARG_VAL,    &append_mode, 1, 0, 0 },
   {"del",              0,  POPT_ARG_NONE,   &delete_during, 0, 0, 0 },
@@ -1236,6 +1245,15 @@ int parse_arguments(int *argc, const cha
 	}
 #endif
 
+#ifndef SUPPORT_PREALLOCATION
+	if (preallocate_files && !am_sender) {
+		snprintf(err_buf, sizeof err_buf,
+			 "preallocation is not supported on this %s\n",
+			 am_server ? "server" : "client");
+		return 0;
+	}
+#endif
+
 	if (write_batch && read_batch) {
 		snprintf(err_buf, sizeof err_buf,
 			"--write-batch and --read-batch can not be used together\n");
@@ -1911,6 +1929,9 @@ void server_options(char **args,int *arg
 	else if (remove_source_files)
 		args[ac++] = "--remove-sent-files";
 
+	if (preallocate_files && am_sender)
+		args[ac++] = "--preallocate";
+
 	*argc = ac;
 	return;
 
--- old/receiver.c
+++ new/receiver.c
@@ -45,6 +45,7 @@ extern int cleanup_got_literal;
 extern int remove_source_files;
 extern int append_mode;
 extern int sparse_files;
+extern int preallocate_files;
 extern int keep_partial;
 extern int checksum_seed;
 extern int inplace;
@@ -138,6 +139,19 @@ static int receive_data(int f_in, char *
 	int32 i;
 	char *map = NULL;
 
+#ifdef SUPPORT_PREALLOCATION
+	int preallocated_len = 0;
+
+	if (preallocate_files && fd != -1 && total_size > 0) {
+		/* Preallocate enough space for file's eventual length if
+		 * possible; seems to reduce fragmentation on Windows. */
+		if (posix_fallocate(fd, 0, total_size) == 0)
+			preallocated_len = total_size;
+		else
+			rsyserr(FINFO, errno, "preallocate %s", full_fname(fname));
+	}
+#endif
+
 	read_sum_head(f_in, &sum);
 
 	if (fd_r >= 0 && size_r > 0) {
@@ -245,8 +259,18 @@ static int receive_data(int f_in, char *
 		goto report_write_error;
 
 #ifdef HAVE_FTRUNCATE
-	if (inplace && fd != -1)
-		ftruncate(fd, offset);
+	/* inplace: New data could be shorter than old data.
+	 * preallocate_files: total_size could have been an overestimate.
+	 *     Cut off any extra preallocated zeros from dest file. */
+	if ((inplace
+#ifdef SUPPORT_PREALLOCATION
+			|| preallocated_len > offset
+#endif
+		) && fd != -1)
+		if (ftruncate(fd, offset) < 0)
+			/* If we fail to truncate, the dest file may be wrong, so we
+			 * must trigger the "partial transfer" error. */
+			rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname));
 #endif
 
 	if (do_progress)
--- old/rsync.h
+++ new/rsync.h
@@ -564,6 +564,10 @@ struct idev_node {
 #define ACLS_NEED_MASK 1
 #endif
 
+#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE
+#define SUPPORT_PREALLOCATION 1
+#endif
+
 #define GID_NONE ((gid_t)-1)
 
 union file_extras {
--- old/rsync.yo
+++ new/rsync.yo
@@ -335,6 +335,7 @@ to the detailed description below for a 
      --super                 receiver attempts super-user activities
      --fake-super            store/recover privileged attrs using xattrs
  -S, --sparse                handle sparse files efficiently
+     --preallocate           posix_fallocate dest files before writing
  -n, --dry-run               show what would have been transferred
  -W, --whole-file            copy files whole (without rsync algorithm)
  -x, --one-file-system       don't cross filesystem boundaries
@@ -956,6 +957,19 @@ NOTE: Don't use this option when the des
 filesystem. It doesn't seem to handle seeks over null regions
 correctly and ends up corrupting the files.
 
+dit(bf(--preallocate)) This tells the receiver to allocate each destination
+file to its eventual size using bf(posix_fallocate)(3) before writing data
+to the file.  If the receiver is remote, this nonstandard option only works
+if the receiver also has the preallocation patch.  Furthermore, this option
+only works if the receiver found the bf(posix_fallocate)(3) call at
+configure time.
+
+Without this option on MS Windows, very large destination files tend to be
+broken into thousands of fragments; advising Windows ahead of time of the
+eventual file size using this option usually reduces the number of
+fragments to one.  However, on Linux, this option appears to just waste
+disk I/O.
+
 dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers,
 instead it will just report the actions it would have taken.
 
--- old/t_stub.c
+++ new/t_stub.c
@@ -22,6 +22,7 @@
 #include "rsync.h"
 
 int modify_window = 0;
+int preallocate_files = 0;
 int module_id = -1;
 int relative_paths = 0;
 int human_readable = 0;
--- old/util.c
+++ new/util.c
@@ -24,6 +24,7 @@
 
 extern int verbose;
 extern int dry_run;
+extern int preallocate_files;
 extern int module_id;
 extern int modify_window;
 extern int relative_paths;
@@ -269,6 +270,10 @@ int copy_file(const char *source, const 
 	int ofd;
 	char buf[1024 * 8];
 	int len;   /* Number of bytes read into `buf'. */
+#ifdef SUPPORT_PREALLOCATION
+	int preallocated_len = 0;
+	int offset = 0;
+#endif
 
 	ifd = do_open(source, O_RDONLY, 0);
 	if (ifd == -1) {
@@ -288,7 +293,27 @@ int copy_file(const char *source, const 
 		return -1;
 	}
 
+#ifdef SUPPORT_PREALLOCATION
+	if (preallocate_files) {
+		/* Preallocate enough space for file's eventual length if
+		 * possible; seems to reduce fragmentation on Windows. */
+		STRUCT_STAT srcst;
+		if (do_fstat(ifd, &srcst) == 0) {
+			if (srcst.st_size > 0) {
+				if (posix_fallocate(ofd, 0, srcst.st_size) == 0)
+					preallocated_len = srcst.st_size;
+				else
+					rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest));
+			}
+		} else
+			rsyserr(FINFO, errno, "fstat %s", full_fname(source));
+	}
+#endif
+
 	while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
+#ifdef SUPPORT_PREALLOCATION
+		offset += len;
+#endif
 		if (full_write(ofd, buf, len) < 0) {
 			rsyserr(FERROR, errno, "write %s", full_fname(dest));
 			close(ifd);
@@ -309,6 +334,16 @@ int copy_file(const char *source, const 
 			full_fname(source));
 	}
 
+#ifdef SUPPORT_PREALLOCATION
+	/* Source file might have shrunk since we fstatted it.
+	 * Cut off any extra preallocated zeros from dest file. */
+	if (preallocated_len > offset)
+		if (ftruncate(ofd, offset) < 0)
+			/* If we fail to truncate, the dest file may be wrong, so we
+			 * must trigger the "partial transfer" error. */
+			rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest));
+#endif
+
 	if (close(ofd) < 0) {
 		rsyserr(FERROR, errno, "close failed on %s",
 			full_fname(dest));