Applying the preallocate patch.

diff --git a/compat.c b/compat.c
index ad29501..45073d3 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -34,6 +34,7 @@ extern int inplace;
 extern int recurse;
 extern int use_qsort;
 extern int allow_inc_recurse;
+extern int preallocate_files;
 extern int append_mode;
 extern int fuzzy_basis;
 extern int read_batch;
@@ -189,6 +190,14 @@ void setup_protocol(int f_out,int f_in)
        if (read_batch)
                check_batch_flags();
 
+#ifndef SUPPORT_PREALLOCATION
+       if (preallocate_files && !am_sender) {
+               rprintf(FERROR, "preallocation is not supported on this %s\n",
+                       am_server ? "Server" : "Client");
+               exit_cleanup(RERR_SYNTAX);
+       }
+#endif
+
        if (protocol_version < 30) {
                if (append_mode == 1)
                        append_mode = 2;

diff --git a/configure.ac b/configure.ac
index 2b5f0a7..73209bf 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -591,13 +591,49 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \
     setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
     seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
     extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
-    initgroups utimensat)
+    initgroups utimensat posix_fallocate)
 
 dnl cygwin iconv.h defines iconv_open as libiconv_open
 if test x"$ac_cv_func_iconv_open" != x"yes"; then
     AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)])
 fi
 
+dnl Preallocation stuff (also fallocate, posix_fallocate function tests above):
+
+AC_CACHE_CHECK([for useable fallocate],rsync_cv_have_fallocate,[
+AC_TRY_LINK([#include <fcntl.h>
+#include <sys/types.h>],
+[fallocate(0, 0, 0, 0);],
+rsync_cv_have_fallocate=yes,rsync_cv_have_fallocate=no)])
+if test x"$rsync_cv_have_fallocate" = x"yes"; then
+    AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error])
+fi
+
+AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[
+AC_TRY_COMPILE([#include <sys/syscall.h>
+#include <sys/types.h>],
+[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);],
+rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)])
+if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then
+    AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number])
+fi
+
+if test x"$ac_cv_func_posix_fallocate" = x"yes"; then
+    AC_MSG_CHECKING([whether posix_fallocate is efficient])
+    case $host_os in
+    *cygwin*)
+       AC_MSG_RESULT(yes)
+       AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1,
+                 [Define if posix_fallocate is efficient (Cygwin)])
+       ;;
+    *)
+       AC_MSG_RESULT(no)
+       ;;
+    esac
+fi
+
+dnl End of preallocation stuff
+
 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
 if test $ac_cv_func_getpgrp = yes; then
     AC_FUNC_GETPGRP

diff --git a/options.c b/options.c
index 3dab4a2..1041c97 100644 (file)
--- a/options.c
+++ b/options.c
@@ -73,6 +73,7 @@ int remove_source_files = 0;
 int one_file_system = 0;
 int protocol_version = PROTOCOL_VERSION;
 int sparse_files = 0;
+int preallocate_files = 0;
 int do_compression = 0;
 int def_compress_level = Z_DEFAULT_COMPRESSION;
 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
@@ -562,6 +563,7 @@ static void print_rsync_version(enum logcode f)
        char const *got_socketpair = "no ";
        char const *have_inplace = "no ";
        char const *hardlinks = "no ";
+       char const *prealloc = "no ";
        char const *symtimes = "no ";
        char const *acls = "no ";
        char const *xattrs = "no ";
@@ -583,6 +585,9 @@ static void print_rsync_version(enum logcode f)
 #ifdef SUPPORT_HARD_LINKS
        hardlinks = "";
 #endif
+#ifdef SUPPORT_PREALLOCATION
+       prealloc = "";
+#endif
 #ifdef SUPPORT_ACLS
        acls = "";
 #endif
@@ -614,8 +619,8 @@ static void print_rsync_version(enum logcode f)
                (int)(sizeof (int64) * 8));
        rprintf(f, "    %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
                got_socketpair, hardlinks, links, ipv6, have_inplace);
-       rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
-               have_inplace, acls, xattrs, iconv, symtimes);
+       rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n",
+               have_inplace, acls, xattrs, iconv, symtimes, prealloc);
 
 #ifdef MAINTAINER_MODE
        rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
@@ -706,6 +711,11 @@ void usage(enum logcode F)
   rprintf(F,"     --fake-super            store/recover privileged attrs using xattrs\n");
 #endif
   rprintf(F," -S, --sparse                handle sparse files efficiently\n");
+#ifdef SUPPORT_PREALLOCATION
+  rprintf(F,"     --preallocate           allocate dest files before writing them\n");
+#else
+  rprintf(F,"     --preallocate           pre-allocate dest files on remote receiver\n");
+#endif
   rprintf(F," -n, --dry-run               perform a trial run with no changes made\n");
   rprintf(F," -W, --whole-file            copy files whole (without delta-xfer algorithm)\n");
   rprintf(F," -x, --one-file-system       don't cross filesystem boundaries\n");
@@ -905,6 +915,7 @@ static struct poptOption long_options[] = {
   {"sparse",          'S', POPT_ARG_VAL,    &sparse_files, 1, 0, 0 },
   {"no-sparse",        0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
   {"no-S",             0,  POPT_ARG_VAL,    &sparse_files, 0, 0, 0 },
+  {"preallocate",      0,  POPT_ARG_NONE,   &preallocate_files, 0, 0, 0},
   {"inplace",          0,  POPT_ARG_VAL,    &inplace, 1, 0, 0 },
   {"no-inplace",       0,  POPT_ARG_VAL,    &inplace, 0, 0, 0 },
   {"append",           0,  POPT_ARG_NONE,   0, OPT_APPEND, 0, 0 },
@@ -2676,6 +2687,9 @@ void server_options(char **args, int *argc_p)
        else if (remove_source_files)
                args[ac++] = "--remove-sent-files";
 
+       if (preallocate_files && am_sender)
+               args[ac++] = "--preallocate";
+
        if (ac > MAX_SERVER_ARGS) { /* Not possible... */
                rprintf(FERROR, "argc overflow in server_options().\n");
                exit_cleanup(RERR_MALLOC);

diff --git a/receiver.c b/receiver.c
index 7641b9b..4c43bc1 100644 (file)
--- a/receiver.c
+++ b/receiver.c
@@ -45,6 +45,7 @@ extern int cleanup_got_literal;
 extern int remove_source_files;
 extern int append_mode;
 extern int sparse_files;
+extern int preallocate_files;
 extern int keep_partial;
 extern int checksum_len;
 extern int checksum_seed;
@@ -227,6 +228,22 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
        char *data;
        int32 i;
        char *map = NULL;
+#ifdef SUPPORT_PREALLOCATION
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+       OFF_T preallocated_len = 0;
+#endif
+
+       if (preallocate_files && fd != -1 && total_size > 0) {
+               /* Try to preallocate enough space for file's eventual length.  Can
+                * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
+               if (do_fallocate(fd, 0, total_size) == 0) {
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+                       preallocated_len = total_size;
+#endif
+               } else
+                       rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
+       }
+#endif
 
        read_sum_head(f_in, &sum);
 
@@ -341,7 +358,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
                goto report_write_error;
 
 #ifdef HAVE_FTRUNCATE
-       if (inplace && fd != -1 && do_ftruncate(fd, offset) < 0) {
+       /* inplace: New data could be shorter than old data.
+        * preallocate_files: total_size could have been an overestimate.
+        *     Cut off any extra preallocated zeros from dest file. */
+       if ((inplace
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+         || preallocated_len > offset
+#endif
+         ) && fd != -1 && do_ftruncate(fd, offset) < 0) {
                rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
                        full_fname(fname));
        }

diff --git a/rsync.h b/rsync.h
index 79402cd..99c7d6c 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -665,6 +665,21 @@ struct ht_int64_node {
 #define ACLS_NEED_MASK 1
 #endif
 
+#if defined HAVE_FALLOCATE || HAVE_SYS_FALLOCATE
+#include <linux/falloc.h>
+#ifdef FALLOC_FL_KEEP_SIZE
+#define SUPPORT_PREALLOCATION 1
+#elif defined HAVE_FTRUNCATE
+#define SUPPORT_PREALLOCATION 1
+#define PREALLOCATE_NEEDS_TRUNCATE 1
+#endif
+#else /* !fallocate */
+#if defined HAVE_EFFICIENT_POSIX_FALLOCATE && defined HAVE_FTRUNCATE
+#define SUPPORT_PREALLOCATION 1
+#define PREALLOCATE_NEEDS_TRUNCATE 1
+#endif
+#endif
+
 union file_extras {
        int32 num;
        uint32 unum;

diff --git a/rsync.yo b/rsync.yo
index 838d968..a1cac98 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -360,6 +360,7 @@ to the detailed description below for a complete description.  verb(
      --super                 receiver attempts super-user activities
      --fake-super            store/recover privileged attrs using xattrs
  -S, --sparse                handle sparse files efficiently
+     --preallocate           allocate dest files before writing
  -n, --dry-run               perform a trial run with no changes made
  -W, --whole-file            copy files whole (w/o delta-xfer algorithm)
  -x, --one-file-system       don't cross filesystem boundaries
@@ -1178,6 +1179,17 @@ dit(bf(-S, --sparse)) Try to handle sparse files efficiently so they take
 up less space on the destination.  Conflicts with bf(--inplace) because it's
 not possible to overwrite data in a sparse fashion.
 
+dit(bf(--preallocate)) This tells the receiver to allocate each destination
+file to its eventual size before writing data to the file.  Rsync will only use
+the real filesystem-level preallocation support provided by Linux's
+bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow
+glibc implementation that writes a zero byte into each block.
+
+Without this option, larger files may not be entirely contiguous on the
+filesystem, but with this option rsync will probably copy more slowly.  If the
+destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS,
+etc.), this option may have no positive effect at all.
+
 dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't
 make any changes (and produces mostly the same output as a real run).  It
 is most commonly used in combination with the bf(-v, --verbose) and/or

diff --git a/syscall.c b/syscall.c
index a99783a..1ed36f3 100644 (file)
--- a/syscall.c
+++ b/syscall.c
@@ -29,6 +29,10 @@
 #include <sys/attr.h>
 #endif
 
+#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE
+#include <sys/syscall.h>
+#endif
+
 extern int dry_run;
 extern int am_root;
 extern int am_sender;
@@ -417,3 +421,25 @@ int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec))
 #else
 #error Need utimes or utime function.
 #endif
+
+#ifdef SUPPORT_PREALLOCATION
+int do_fallocate(int fd, OFF_T offset, OFF_T length)
+{
+#ifdef FALLOC_FL_KEEP_SIZE
+#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE
+#else
+#define DO_FALLOC_OPTIONS 0
+#endif
+       RETURN_ERROR_IF(dry_run, 0);
+       RETURN_ERROR_IF_RO_OR_LO;
+#if defined HAVE_FALLOCATE
+       return fallocate(fd, DO_FALLOC_OPTIONS, offset, length);
+#elif defined HAVE_SYS_FALLOCATE
+       return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length);
+#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE
+       return posix_fallocate(fd, offset, length);
+#else
+#error Coding error in SUPPORT_PREALLOCATION logic.
+#endif
+}
+#endif

diff --git a/t_stub.c b/t_stub.c
index 3bfa699..bbfdee7 100644 (file)
--- a/t_stub.c
+++ b/t_stub.c
@@ -22,6 +22,7 @@
 #include "rsync.h"
 
 int modify_window = 0;
+int preallocate_files = 0;
 int module_id = -1;
 int relative_paths = 0;
 int module_dirlen = 0;

diff --git a/util.c b/util.c
index a53ea29..3f611d1 100644 (file)
--- a/util.c
+++ b/util.c
@@ -30,6 +30,7 @@ extern int modify_window;
 extern int relative_paths;
 extern int preserve_times;
 extern int preserve_xattrs;
+extern int preallocate_files;
 extern char *module_dir;
 extern unsigned int module_dirlen;
 extern char *partial_dir;
@@ -315,6 +316,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
        int ifd;
        char buf[1024 * 8];
        int len;   /* Number of bytes read into `buf'. */
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+       OFF_T preallocated_len = 0, offset = 0;
+#endif
 
        if ((ifd = do_open(source, O_RDONLY, 0)) < 0) {
                int save_errno = errno;
@@ -345,6 +349,25 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
                }
        }
 
+#ifdef SUPPORT_PREALLOCATION
+       if (preallocate_files) {
+               STRUCT_STAT srcst;
+
+               /* Try to preallocate enough space for file's eventual length.  Can
+                * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
+               if (do_fstat(ifd, &srcst) < 0)
+                       rsyserr(FWARNING, errno, "fstat %s", full_fname(source));
+               else if (srcst.st_size > 0) {
+                       if (do_fallocate(ofd, 0, srcst.st_size) == 0) {
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+                               preallocated_len = srcst.st_size;
+#endif
+                       } else
+                               rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest));
+               }
+       }
+#endif
+
        while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
                if (full_write(ofd, buf, len) < 0) {
                        int save_errno = errno;
@@ -354,6 +377,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
                        errno = save_errno;
                        return -1;
                }
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+               offset += len;
+#endif
        }
 
        if (len < 0) {
@@ -370,6 +396,16 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode)
                        full_fname(source));
        }
 
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+       /* Source file might have shrunk since we fstatted it.
+        * Cut off any extra preallocated zeros from dest file. */
+       if (offset < preallocated_len && do_ftruncate(ofd, offset) < 0) {
+               /* If we fail to truncate, the dest file may be wrong, so we
+                * must trigger the "partial transfer" error. */
+               rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest));
+       }
+#endif
+
        if (close(ofd) < 0) {
                int save_errno = errno;
                rsyserr(FERROR_XFER, errno, "close failed on %s",