From: Wayne Davison Date: Tue, 5 Apr 2011 04:44:12 +0000 (-0700) Subject: Applying the preallocate patch. X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/commitdiff_plain/28b519c93b6db30b6520d46f8cd65160213fddd2 Applying the preallocate patch. --- diff --git a/compat.c b/compat.c index ad29501e..45073d3d 100644 --- a/compat.c +++ b/compat.c @@ -34,6 +34,7 @@ extern int inplace; extern int recurse; extern int use_qsort; extern int allow_inc_recurse; +extern int preallocate_files; extern int append_mode; extern int fuzzy_basis; extern int read_batch; @@ -189,6 +190,14 @@ void setup_protocol(int f_out,int f_in) if (read_batch) check_batch_flags(); +#ifndef SUPPORT_PREALLOCATION + if (preallocate_files && !am_sender) { + rprintf(FERROR, "preallocation is not supported on this %s\n", + am_server ? "Server" : "Client"); + exit_cleanup(RERR_SYNTAX); + } +#endif + if (protocol_version < 30) { if (append_mode == 1) append_mode = 2; diff --git a/configure.ac b/configure.ac index 2b5f0a7b..73209bf0 100644 --- a/configure.ac +++ b/configure.ac @@ -591,13 +591,49 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ - initgroups utimensat) + initgroups utimensat posix_fallocate) dnl cygwin iconv.h defines iconv_open as libiconv_open if test x"$ac_cv_func_iconv_open" != x"yes"; then AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) fi +dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): + +AC_CACHE_CHECK([for useable fallocate],rsync_cv_have_fallocate,[ +AC_TRY_LINK([#include +#include ], +[fallocate(0, 0, 0, 0);], +rsync_cv_have_fallocate=yes,rsync_cv_have_fallocate=no)]) +if test x"$rsync_cv_have_fallocate" = x"yes"; then + AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error]) +fi + +AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ +AC_TRY_COMPILE([#include +#include ], +[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);], +rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) +if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then + AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) +fi + +if test x"$ac_cv_func_posix_fallocate" = x"yes"; then + AC_MSG_CHECKING([whether posix_fallocate is efficient]) + case $host_os in + *cygwin*) + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, + [Define if posix_fallocate is efficient (Cygwin)]) + ;; + *) + AC_MSG_RESULT(no) + ;; + esac +fi + +dnl End of preallocation stuff + AC_CHECK_FUNCS(getpgrp tcgetpgrp) if test $ac_cv_func_getpgrp = yes; then AC_FUNC_GETPGRP diff --git a/options.c b/options.c index 3dab4a24..1041c975 100644 --- a/options.c +++ b/options.c @@ -73,6 +73,7 @@ int remove_source_files = 0; int one_file_system = 0; int protocol_version = PROTOCOL_VERSION; int sparse_files = 0; +int preallocate_files = 0; int do_compression = 0; int def_compress_level = Z_DEFAULT_COMPRESSION; int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ @@ -562,6 +563,7 @@ static void print_rsync_version(enum logcode f) char const *got_socketpair = "no "; char const *have_inplace = "no "; char const *hardlinks = "no "; + char const *prealloc = "no "; char const *symtimes = "no "; char const *acls = "no "; char const *xattrs = "no "; @@ -583,6 +585,9 @@ static void print_rsync_version(enum logcode f) #ifdef SUPPORT_HARD_LINKS hardlinks = ""; #endif +#ifdef SUPPORT_PREALLOCATION + prealloc = ""; +#endif #ifdef SUPPORT_ACLS acls = ""; #endif @@ -614,8 +619,8 @@ static void print_rsync_version(enum logcode f) (int)(sizeof (int64) * 8)); rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", got_socketpair, hardlinks, links, ipv6, have_inplace); - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", - have_inplace, acls, xattrs, iconv, symtimes); + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n", + have_inplace, acls, xattrs, iconv, symtimes, prealloc); #ifdef MAINTAINER_MODE rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); @@ -706,6 +711,11 @@ void usage(enum logcode F) rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); #endif rprintf(F," -S, --sparse handle sparse files efficiently\n"); +#ifdef SUPPORT_PREALLOCATION + rprintf(F," --preallocate allocate dest files before writing them\n"); +#else + rprintf(F," --preallocate pre-allocate dest files on remote receiver\n"); +#endif rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); @@ -905,6 +915,7 @@ static struct poptOption long_options[] = { {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, @@ -2676,6 +2687,9 @@ void server_options(char **args, int *argc_p) else if (remove_source_files) args[ac++] = "--remove-sent-files"; + if (preallocate_files && am_sender) + args[ac++] = "--preallocate"; + if (ac > MAX_SERVER_ARGS) { /* Not possible... */ rprintf(FERROR, "argc overflow in server_options().\n"); exit_cleanup(RERR_MALLOC); diff --git a/receiver.c b/receiver.c index 7641b9b5..4c43bc16 100644 --- a/receiver.c +++ b/receiver.c @@ -45,6 +45,7 @@ extern int cleanup_got_literal; extern int remove_source_files; extern int append_mode; extern int sparse_files; +extern int preallocate_files; extern int keep_partial; extern int checksum_len; extern int checksum_seed; @@ -227,6 +228,22 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, char *data; int32 i; char *map = NULL; +#ifdef SUPPORT_PREALLOCATION +#ifdef PREALLOCATE_NEEDS_TRUNCATE + OFF_T preallocated_len = 0; +#endif + + if (preallocate_files && fd != -1 && total_size > 0) { + /* Try to preallocate enough space for file's eventual length. Can + * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ + if (do_fallocate(fd, 0, total_size) == 0) { +#ifdef PREALLOCATE_NEEDS_TRUNCATE + preallocated_len = total_size; +#endif + } else + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); + } +#endif read_sum_head(f_in, &sum); @@ -341,7 +358,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, goto report_write_error; #ifdef HAVE_FTRUNCATE - if (inplace && fd != -1 && do_ftruncate(fd, offset) < 0) { + /* inplace: New data could be shorter than old data. + * preallocate_files: total_size could have been an overestimate. + * Cut off any extra preallocated zeros from dest file. */ + if ((inplace +#ifdef PREALLOCATE_NEEDS_TRUNCATE + || preallocated_len > offset +#endif + ) && fd != -1 && do_ftruncate(fd, offset) < 0) { rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", full_fname(fname)); } diff --git a/rsync.h b/rsync.h index 79402cd7..99c7d6c4 100644 --- a/rsync.h +++ b/rsync.h @@ -665,6 +665,21 @@ struct ht_int64_node { #define ACLS_NEED_MASK 1 #endif +#if defined HAVE_FALLOCATE || HAVE_SYS_FALLOCATE +#include +#ifdef FALLOC_FL_KEEP_SIZE +#define SUPPORT_PREALLOCATION 1 +#elif defined HAVE_FTRUNCATE +#define SUPPORT_PREALLOCATION 1 +#define PREALLOCATE_NEEDS_TRUNCATE 1 +#endif +#else /* !fallocate */ +#if defined HAVE_EFFICIENT_POSIX_FALLOCATE && defined HAVE_FTRUNCATE +#define SUPPORT_PREALLOCATION 1 +#define PREALLOCATE_NEEDS_TRUNCATE 1 +#endif +#endif + union file_extras { int32 num; uint32 unum; diff --git a/rsync.yo b/rsync.yo index 838d9688..a1cac988 100644 --- a/rsync.yo +++ b/rsync.yo @@ -360,6 +360,7 @@ to the detailed description below for a complete description. verb( --super receiver attempts super-user activities --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently + --preallocate allocate dest files before writing -n, --dry-run perform a trial run with no changes made -W, --whole-file copy files whole (w/o delta-xfer algorithm) -x, --one-file-system don't cross filesystem boundaries @@ -1178,6 +1179,17 @@ dit(bf(-S, --sparse)) Try to handle sparse files efficiently so they take up less space on the destination. Conflicts with bf(--inplace) because it's not possible to overwrite data in a sparse fashion. +dit(bf(--preallocate)) This tells the receiver to allocate each destination +file to its eventual size before writing data to the file. Rsync will only use +the real filesystem-level preallocation support provided by Linux's +bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow +glibc implementation that writes a zero byte into each block. + +Without this option, larger files may not be entirely contiguous on the +filesystem, but with this option rsync will probably copy more slowly. If the +destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS, +etc.), this option may have no positive effect at all. + dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't make any changes (and produces mostly the same output as a real run). It is most commonly used in combination with the bf(-v, --verbose) and/or diff --git a/syscall.c b/syscall.c index a99783a3..1ed36f3c 100644 --- a/syscall.c +++ b/syscall.c @@ -29,6 +29,10 @@ #include #endif +#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE +#include +#endif + extern int dry_run; extern int am_root; extern int am_sender; @@ -417,3 +421,25 @@ int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)) #else #error Need utimes or utime function. #endif + +#ifdef SUPPORT_PREALLOCATION +int do_fallocate(int fd, OFF_T offset, OFF_T length) +{ +#ifdef FALLOC_FL_KEEP_SIZE +#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE +#else +#define DO_FALLOC_OPTIONS 0 +#endif + RETURN_ERROR_IF(dry_run, 0); + RETURN_ERROR_IF_RO_OR_LO; +#if defined HAVE_FALLOCATE + return fallocate(fd, DO_FALLOC_OPTIONS, offset, length); +#elif defined HAVE_SYS_FALLOCATE + return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length); +#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE + return posix_fallocate(fd, offset, length); +#else +#error Coding error in SUPPORT_PREALLOCATION logic. +#endif +} +#endif diff --git a/t_stub.c b/t_stub.c index 3bfa699d..bbfdee71 100644 --- a/t_stub.c +++ b/t_stub.c @@ -22,6 +22,7 @@ #include "rsync.h" int modify_window = 0; +int preallocate_files = 0; int module_id = -1; int relative_paths = 0; int module_dirlen = 0; diff --git a/util.c b/util.c index a53ea291..3f611d15 100644 --- a/util.c +++ b/util.c @@ -30,6 +30,7 @@ extern int modify_window; extern int relative_paths; extern int preserve_times; extern int preserve_xattrs; +extern int preallocate_files; extern char *module_dir; extern unsigned int module_dirlen; extern char *partial_dir; @@ -315,6 +316,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) int ifd; char buf[1024 * 8]; int len; /* Number of bytes read into `buf'. */ +#ifdef PREALLOCATE_NEEDS_TRUNCATE + OFF_T preallocated_len = 0, offset = 0; +#endif if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { int save_errno = errno; @@ -345,6 +349,25 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) } } +#ifdef SUPPORT_PREALLOCATION + if (preallocate_files) { + STRUCT_STAT srcst; + + /* Try to preallocate enough space for file's eventual length. Can + * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ + if (do_fstat(ifd, &srcst) < 0) + rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); + else if (srcst.st_size > 0) { + if (do_fallocate(ofd, 0, srcst.st_size) == 0) { +#ifdef PREALLOCATE_NEEDS_TRUNCATE + preallocated_len = srcst.st_size; +#endif + } else + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); + } + } +#endif + while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { if (full_write(ofd, buf, len) < 0) { int save_errno = errno; @@ -354,6 +377,9 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) errno = save_errno; return -1; } +#ifdef PREALLOCATE_NEEDS_TRUNCATE + offset += len; +#endif } if (len < 0) { @@ -370,6 +396,16 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) full_fname(source)); } +#ifdef PREALLOCATE_NEEDS_TRUNCATE + /* Source file might have shrunk since we fstatted it. + * Cut off any extra preallocated zeros from dest file. */ + if (offset < preallocated_len && do_ftruncate(ofd, offset) < 0) { + /* If we fail to truncate, the dest file may be wrong, so we + * must trigger the "partial transfer" error. */ + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); + } +#endif + if (close(ofd) < 0) { int save_errno = errno; rsyserr(FERROR_XFER, errno, "close failed on %s",