X-Git-Url: https://mattmccutchen.net/rsync/rsync-patches.git/blobdiff_plain/5795bf59eae77f5d56b33a883708280b458f9fca..c1ff70aa47e11c5b37634479a0facee775a7b6d9:/preallocate.diff diff --git a/preallocate.diff b/preallocate.diff index 3481313..042d983 100644 --- a/preallocate.diff +++ b/preallocate.diff @@ -9,197 +9,305 @@ To use this patch, run these commands for a successful build: ./configure make ---- old/configure.in -+++ new/configure.in -@@ -560,7 +560,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd - strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ +based-on: a01e3b490eb36ccf9e704840e1b6683dab867550 +diff --git a/compat.c b/compat.c +--- a/compat.c ++++ b/compat.c +@@ -33,6 +33,7 @@ extern int inplace; + extern int recurse; + extern int use_qsort; + extern int allow_inc_recurse; ++extern int preallocate_files; + extern int append_mode; + extern int fuzzy_basis; + extern int read_batch; +@@ -188,6 +189,14 @@ void setup_protocol(int f_out,int f_in) + if (read_batch) + check_batch_flags(); + ++#ifndef SUPPORT_PREALLOCATION ++ if (preallocate_files && !am_sender) { ++ rprintf(FERROR, "preallocation is not supported on this %s\n", ++ am_server ? "Server" : "Client"); ++ exit_cleanup(RERR_SYNTAX); ++ } ++#endif ++ + if (protocol_version < 30) { + if (append_mode == 1) + append_mode = 2; +diff --git a/configure.in b/configure.in +--- a/configure.in ++++ b/configure.in +@@ -589,13 +589,49 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ - strerror putenv iconv_open locale_charset nl_langinfo \ -- sigaction sigprocmask) -+ sigaction sigprocmask posix_fallocate) + seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \ + extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ +- initgroups utimensat) ++ initgroups utimensat posix_fallocate) + dnl cygwin iconv.h defines iconv_open as libiconv_open + if test x"$ac_cv_func_iconv_open" != x"yes"; then + AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) + fi + ++dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): ++ ++AC_CACHE_CHECK([for useable fallocate],rsync_cv_have_fallocate,[ ++AC_TRY_LINK([#include ++#include ], ++[fallocate(0, 0, 0, 0);], ++rsync_cv_have_fallocate=yes,rsync_cv_have_fallocate=no)]) ++if test x"$rsync_cv_have_fallocate" = x"yes"; then ++ AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error]) ++fi ++ ++AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ ++AC_TRY_COMPILE([#include ++#include ], ++[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);], ++rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) ++if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then ++ AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) ++fi ++ ++if test x"$ac_cv_func_posix_fallocate" = x"yes"; then ++ AC_MSG_CHECKING([whether posix_fallocate is efficient]) ++ case $host_os in ++ *cygwin*) ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, ++ [Define if posix_fallocate is efficient (Cygwin)]) ++ ;; ++ *) ++ AC_MSG_RESULT(no) ++ ;; ++ esac ++fi ++ ++dnl End of preallocation stuff ++ AC_CHECK_FUNCS(getpgrp tcgetpgrp) if test $ac_cv_func_getpgrp = yes; then ---- old/options.c -+++ new/options.c -@@ -70,6 +70,7 @@ int remove_source_files = 0; + AC_FUNC_GETPGRP +diff --git a/options.c b/options.c +--- a/options.c ++++ b/options.c +@@ -73,6 +73,7 @@ int remove_source_files = 0; int one_file_system = 0; int protocol_version = PROTOCOL_VERSION; int sparse_files = 0; +int preallocate_files = 0; int do_compression = 0; int def_compress_level = Z_DEFAULT_COMPRESSION; - int am_root = 0; -@@ -205,6 +206,7 @@ static void print_rsync_version(enum log - char const *xattrs = "no "; + int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ +@@ -567,6 +568,7 @@ static void print_rsync_version(enum logcode f) char const *links = "no "; + char const *iconv = "no "; char const *ipv6 = "no "; + char const *preallocation = "no "; STRUCT_STAT *dumstat; - #ifdef HAVE_SOCKETPAIR -@@ -233,6 +235,10 @@ static void print_rsync_version(enum log - ipv6 = ""; + #if SUBPROTOCOL_VERSION != 0 +@@ -600,6 +602,9 @@ static void print_rsync_version(enum logcode f) + #ifdef CAN_SET_SYMLINK_TIMES + symtimes = ""; #endif - +#ifdef SUPPORT_PREALLOCATION + preallocation = ""; +#endif -+ - rprintf(f, "%s version %s protocol version %d\n", - RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION); - rprintf(f, "Copyright (C) 1996-2007 by Andrew Tridgell, Wayne Davison, and others.\n"); -@@ -243,8 +249,8 @@ static void print_rsync_version(enum log + + rprintf(f, "%s version %s protocol version %d%s\n", + RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); +@@ -613,8 +618,8 @@ static void print_rsync_version(enum logcode f) (int)(sizeof (int64) * 8)); rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", got_socketpair, hardlinks, links, ipv6, have_inplace); -- rprintf(f, " %sappend, %sACLs, %sxattrs\n", -- have_inplace, acls, xattrs); -+ rprintf(f, " %sappend, %sACLs, %sxattrs, %spreallocation\n", -+ have_inplace, acls, xattrs, preallocation); +- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", +- have_inplace, acls, xattrs, iconv, symtimes); ++ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %spreallocation\n", ++ have_inplace, acls, xattrs, iconv, symtimes, preallocation); #ifdef MAINTAINER_MODE rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); -@@ -327,6 +333,9 @@ void usage(enum logcode F) - rprintf(F," -O, --omit-dir-times omit directories when preserving times\n"); - rprintf(F," --super receiver attempts super-user activities\n"); +@@ -704,6 +709,11 @@ void usage(enum logcode F) + rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); + #endif rprintf(F," -S, --sparse handle sparse files efficiently\n"); +#ifdef SUPPORT_PREALLOCATION -+ rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); ++ rprintf(F," --preallocate allocate dest files before writing them\n"); ++#else ++ rprintf(F," --preallocate pre-allocate dest files on remote receiver\n"); +#endif - rprintf(F," -n, --dry-run show what would have been transferred\n"); - rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n"); + rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); + rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); -@@ -494,6 +503,7 @@ static struct poptOption long_options[] - {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, - {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, - {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, +@@ -900,6 +910,7 @@ static struct poptOption long_options[] = { + {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, + {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, + {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, - {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, - {"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 }, - {"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 }, -@@ -1187,6 +1197,15 @@ int parse_arguments(int *argc, const cha - } - #endif - -+#ifndef SUPPORT_PREALLOCATION -+ if (preallocate_files && !am_sender) { -+ snprintf(err_buf, sizeof err_buf, -+ "preallocation is not supported on this %s\n", -+ am_server ? "server" : "client"); -+ return 0; -+ } -+#endif -+ - if (write_batch && read_batch) { - snprintf(err_buf, sizeof err_buf, - "--write-batch and --read-batch can not be used together\n"); -@@ -1838,6 +1857,9 @@ void server_options(char **args,int *arg + {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, + {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, + {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, +@@ -2661,6 +2672,9 @@ void server_options(char **args, int *argc_p) else if (remove_source_files) args[ac++] = "--remove-sent-files"; + if (preallocate_files && am_sender) + args[ac++] = "--preallocate"; + - *argc = ac; - return; - ---- old/receiver.c -+++ new/receiver.c -@@ -45,6 +45,7 @@ extern int cleanup_got_literal; + if (ac > MAX_SERVER_ARGS) { /* Not possible... */ + rprintf(FERROR, "argc overflow in server_options().\n"); + exit_cleanup(RERR_MALLOC); +diff --git a/receiver.c b/receiver.c +--- a/receiver.c ++++ b/receiver.c +@@ -44,6 +44,7 @@ extern int cleanup_got_literal; extern int remove_source_files; extern int append_mode; extern int sparse_files; +extern int preallocate_files; extern int keep_partial; + extern int checksum_len; extern int checksum_seed; - extern int inplace; -@@ -138,6 +139,19 @@ static int receive_data(int f_in, char * +@@ -207,6 +208,22 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, + char *data; int32 i; char *map = NULL; - +#ifdef SUPPORT_PREALLOCATION -+ int preallocated_len = 0; ++#ifdef PREALLOCATE_NEEDS_TRUNCATE ++ OFF_T preallocated_len = 0; ++#endif + + if (preallocate_files && fd != -1 && total_size > 0) { -+ /* Preallocate enough space for file's eventual length if -+ * possible; seems to reduce fragmentation on Windows. */ -+ if (posix_fallocate(fd, 0, total_size) == 0) ++ /* Try to preallocate enough space for file's eventual length. Can ++ * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ ++ if (do_fallocate(fd, 0, total_size) == 0) { ++#ifdef PREALLOCATE_NEEDS_TRUNCATE + preallocated_len = total_size; -+ else -+ rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); ++#endif ++ } else ++ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); + } +#endif -+ + read_sum_head(f_in, &sum); - if (fd_r >= 0 && size_r > 0) { -@@ -245,8 +259,18 @@ static int receive_data(int f_in, char * +@@ -317,8 +334,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, goto report_write_error; #ifdef HAVE_FTRUNCATE -- if (inplace && fd != -1) -- ftruncate(fd, offset); +- if (inplace && fd != -1 +- && ftruncate(fd, offset) < 0) { + /* inplace: New data could be shorter than old data. + * preallocate_files: total_size could have been an overestimate. + * Cut off any extra preallocated zeros from dest file. */ + if ((inplace -+#ifdef SUPPORT_PREALLOCATION -+ || preallocated_len > offset ++#ifdef PREALLOCATE_NEEDS_TRUNCATE ++ || preallocated_len > offset +#endif -+ ) && fd != -1) -+ if (ftruncate(fd, offset) < 0) -+ /* If we fail to truncate, the dest file may be wrong, so we -+ * must trigger the "partial transfer" error. */ -+ rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname)); - #endif - - if (do_progress) ---- old/rsync.h -+++ new/rsync.h -@@ -549,6 +549,10 @@ struct idev_node { ++ ) && fd != -1 && ftruncate(fd, offset) < 0) { + rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", + full_fname(fname)); + } +diff --git a/rsync.h b/rsync.h +--- a/rsync.h ++++ b/rsync.h +@@ -646,6 +646,21 @@ struct ht_int64_node { #define ACLS_NEED_MASK 1 #endif -+#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE ++#if defined HAVE_FALLOCATE || HAVE_SYS_FALLOCATE ++#include ++#ifdef FALLOC_FL_KEEP_SIZE ++#define SUPPORT_PREALLOCATION 1 ++#elif defined HAVE_FTRUNCATE ++#define SUPPORT_PREALLOCATION 1 ++#define PREALLOCATE_NEEDS_TRUNCATE 1 ++#endif ++#else /* !fallocate */ ++#if defined HAVE_EFFICIENT_POSIX_FALLOCATE && defined HAVE_FTRUNCATE +#define SUPPORT_PREALLOCATION 1 ++#define PREALLOCATE_NEEDS_TRUNCATE 1 ++#endif +#endif + - #define GID_NONE ((gid_t)-1) - union file_extras { ---- old/rsync.yo -+++ new/rsync.yo -@@ -334,6 +334,7 @@ to the detailed description below for a - -O, --omit-dir-times omit directories when preserving times + int32 num; + uint32 unum; +diff --git a/rsync.yo b/rsync.yo +--- a/rsync.yo ++++ b/rsync.yo +@@ -359,6 +359,7 @@ to the detailed description below for a complete description. verb( --super receiver attempts super-user activities + --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently -+ --preallocate posix_fallocate dest files before writing - -n, --dry-run show what would have been transferred - -W, --whole-file copy files whole (without rsync algorithm) ++ --preallocate allocate dest files before writing + -n, --dry-run perform a trial run with no changes made + -W, --whole-file copy files whole (w/o delta-xfer algorithm) -x, --one-file-system don't cross filesystem boundaries -@@ -920,6 +921,19 @@ NOTE: Don't use this option when the des - filesystem. It doesn't seem to handle seeks over null regions - correctly and ends up corrupting the files. +@@ -1127,6 +1128,17 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" + filesystem. It seems to have problems seeking over null regions, + and ends up corrupting the files. +dit(bf(--preallocate)) This tells the receiver to allocate each destination -+file to its eventual size using bf(posix_fallocate)(3) before writing data -+to the file. If the receiver is remote, this nonstandard option only works -+if the receiver also has the preallocation patch. Furthermore, this option -+only works if the receiver found the bf(posix_fallocate)(3) call at -+configure time. ++file to its eventual size before writing data to the file. Rsync will only use ++the real filesystem-level preallocation support provided by Linux's ++bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow ++glibc implementation that writes a zero byte into each block. + -+Without this option on MS Windows, very large destination files tend to be -+broken into thousands of fragments; advising Windows ahead of time of the -+eventual file size using this option usually reduces the number of -+fragments to one. However, on Linux, this option appears to just waste -+disk I/O. ++Without this option, larger files may not be entirely contiguous on the ++filesystem, but with this option rsync will probably copy more slowly. If the ++destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS, ++etc.), this option may have no positive effect at all. + - dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers, - instead it will just report the actions it would have taken. + dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't + make any changes (and produces mostly the same output as a real run). It + is most commonly used in combination with the bf(-v, --verbose) and/or +diff --git a/syscall.c b/syscall.c +--- a/syscall.c ++++ b/syscall.c +@@ -29,6 +29,10 @@ + #include + #endif ---- old/t_stub.c -+++ new/t_stub.c ++#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE ++#include ++#endif ++ + extern int dry_run; + extern int am_root; + extern int am_sender; +@@ -325,3 +329,25 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence) + return lseek(fd, offset, whence); + #endif + } ++ ++#ifdef SUPPORT_PREALLOCATION ++int do_fallocate(int fd, OFF_T offset, OFF_T length) ++{ ++#ifdef FALLOC_FL_KEEP_SIZE ++#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE ++#else ++#define DO_FALLOC_OPTIONS 0 ++#endif ++ RETURN_ERROR_IF(dry_run, 0); ++ RETURN_ERROR_IF_RO_OR_LO; ++#if defined HAVE_FALLOCATE ++ return fallocate(fd, DO_FALLOC_OPTIONS, offset, length); ++#elif defined HAVE_SYS_FALLOCATE ++ return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length); ++#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE ++ return posix_fallocate(fd, offset, length); ++#else ++#error Coding error in SUPPORT_PREALLOCATION logic. ++#endif ++} ++#endif +diff --git a/t_stub.c b/t_stub.c +--- a/t_stub.c ++++ b/t_stub.c @@ -22,6 +22,7 @@ #include "rsync.h" @@ -207,70 +315,78 @@ To use this patch, run these commands for a successful build: +int preallocate_files = 0; int module_id = -1; int relative_paths = 0; - int human_readable = 0; ---- old/util.c -+++ new/util.c -@@ -24,6 +24,7 @@ + int module_dirlen = 0; +diff --git a/util.c b/util.c +--- a/util.c ++++ b/util.c +@@ -26,6 +26,7 @@ + #include "inums.h" - extern int verbose; extern int dry_run; +extern int preallocate_files; extern int module_id; extern int modify_window; extern int relative_paths; -@@ -269,6 +270,10 @@ int copy_file(const char *source, const - int ofd; +@@ -332,6 +333,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) + int ifd; char buf[1024 * 8]; int len; /* Number of bytes read into `buf'. */ -+#ifdef SUPPORT_PREALLOCATION -+ int preallocated_len = 0; -+ int offset = 0; ++#ifdef PREALLOCATE_NEEDS_TRUNCATE ++ OFF_T preallocated_len = 0, offset = 0; +#endif - ifd = do_open(source, O_RDONLY, 0); - if (ifd == -1) { -@@ -288,7 +293,27 @@ int copy_file(const char *source, const - return -1; + if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { + int save_errno = errno; +@@ -357,6 +361,25 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) + } } +#ifdef SUPPORT_PREALLOCATION + if (preallocate_files) { -+ /* Preallocate enough space for file's eventual length if -+ * possible; seems to reduce fragmentation on Windows. */ + STRUCT_STAT srcst; -+ if (do_fstat(ifd, &srcst) == 0) { -+ if (srcst.st_size > 0) { -+ if (posix_fallocate(ofd, 0, srcst.st_size) == 0) -+ preallocated_len = srcst.st_size; -+ else -+ rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); -+ } -+ } else -+ rsyserr(FINFO, errno, "fstat %s", full_fname(source)); ++ ++ /* Try to preallocate enough space for file's eventual length. Can ++ * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ ++ if (do_fstat(ifd, &srcst) < 0) ++ rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); ++ else if (srcst.st_size > 0) { ++ if (do_fallocate(ofd, 0, srcst.st_size) == 0) { ++#ifdef PREALLOCATE_NEEDS_TRUNCATE ++ preallocated_len = srcst.st_size; ++#endif ++ } else ++ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); ++ } + } +#endif + while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { -+#ifdef SUPPORT_PREALLOCATION + if (full_write(ofd, buf, len) < 0) { + int save_errno = errno; +@@ -366,6 +389,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) + errno = save_errno; + return -1; + } ++#ifdef PREALLOCATE_NEEDS_TRUNCATE + offset += len; +#endif - if (full_write(ofd, buf, len) < 0) { - rsyserr(FERROR, errno, "write %s", full_fname(dest)); - close(ifd); -@@ -309,6 +334,16 @@ int copy_file(const char *source, const + } + + if (len < 0) { +@@ -382,6 +408,16 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) full_fname(source)); } -+#ifdef SUPPORT_PREALLOCATION ++#ifdef PREALLOCATE_NEEDS_TRUNCATE + /* Source file might have shrunk since we fstatted it. + * Cut off any extra preallocated zeros from dest file. */ -+ if (preallocated_len > offset) -+ if (ftruncate(ofd, offset) < 0) -+ /* If we fail to truncate, the dest file may be wrong, so we -+ * must trigger the "partial transfer" error. */ -+ rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest)); ++ if (offset < preallocated_len && ftruncate(ofd, offset) < 0) { ++ /* If we fail to truncate, the dest file may be wrong, so we ++ * must trigger the "partial transfer" error. */ ++ rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); ++ } +#endif + if (close(ofd) < 0) { - rsyserr(FERROR, errno, "close failed on %s", - full_fname(dest)); + int save_errno = errno; + rsyserr(FERROR_XFER, errno, "close failed on %s",