extern int recurse;
extern int use_qsort;
extern int allow_inc_recurse;
+extern int preallocate_files;
extern int append_mode;
extern int fuzzy_basis;
extern int read_batch;
if (read_batch)
check_batch_flags();
+#ifndef SUPPORT_PREALLOCATION
+ if (preallocate_files && !am_sender) {
+ rprintf(FERROR, "preallocation is not supported on this %s\n",
+ am_server ? "Server" : "Client");
+ exit_cleanup(RERR_SYNTAX);
+ }
+#endif
+
if (protocol_version < 30) {
if (append_mode == 1)
append_mode = 2;
setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
- initgroups utimensat)
+ initgroups utimensat posix_fallocate)
dnl cygwin iconv.h defines iconv_open as libiconv_open
if test x"$ac_cv_func_iconv_open" != x"yes"; then
AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)])
fi
+dnl Preallocation stuff (also fallocate, posix_fallocate function tests above):
+
+AC_CACHE_CHECK([for useable fallocate],rsync_cv_have_fallocate,[
+AC_TRY_LINK([#include <fcntl.h>
+#include <sys/types.h>],
+[fallocate(0, 0, 0, 0);],
+rsync_cv_have_fallocate=yes,rsync_cv_have_fallocate=no)])
+if test x"$rsync_cv_have_fallocate" = x"yes"; then
+ AC_DEFINE(HAVE_FALLOCATE, 1, [Define to 1 if you have the fallocate function and it compiles and links without error])
+fi
+
+AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[
+AC_TRY_COMPILE([#include <sys/syscall.h>
+#include <sys/types.h>],
+[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);],
+rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)])
+if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then
+ AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number])
+fi
+
+if test x"$ac_cv_func_posix_fallocate" = x"yes"; then
+ AC_MSG_CHECKING([whether posix_fallocate is efficient])
+ case $host_os in
+ *cygwin*)
+ AC_MSG_RESULT(yes)
+ AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1,
+ [Define if posix_fallocate is efficient (Cygwin)])
+ ;;
+ *)
+ AC_MSG_RESULT(no)
+ ;;
+ esac
+fi
+
+dnl End of preallocation stuff
+
AC_CHECK_FUNCS(getpgrp tcgetpgrp)
if test $ac_cv_func_getpgrp = yes; then
AC_FUNC_GETPGRP
int one_file_system = 0;
int protocol_version = PROTOCOL_VERSION;
int sparse_files = 0;
+int preallocate_files = 0;
int do_compression = 0;
int def_compress_level = Z_DEFAULT_COMPRESSION;
int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
char const *got_socketpair = "no ";
char const *have_inplace = "no ";
char const *hardlinks = "no ";
+ char const *prealloc = "no ";
char const *symtimes = "no ";
char const *acls = "no ";
char const *xattrs = "no ";
#ifdef SUPPORT_HARD_LINKS
hardlinks = "";
#endif
+#ifdef SUPPORT_PREALLOCATION
+ prealloc = "";
+#endif
#ifdef SUPPORT_ACLS
acls = "";
#endif
(int)(sizeof (int64) * 8));
rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
got_socketpair, hardlinks, links, ipv6, have_inplace);
- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
- have_inplace, acls, xattrs, iconv, symtimes);
+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n",
+ have_inplace, acls, xattrs, iconv, symtimes, prealloc);
#ifdef MAINTAINER_MODE
rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
rprintf(F," --fake-super store/recover privileged attrs using xattrs\n");
#endif
rprintf(F," -S, --sparse handle sparse files efficiently\n");
+#ifdef SUPPORT_PREALLOCATION
+ rprintf(F," --preallocate allocate dest files before writing them\n");
+#else
+ rprintf(F," --preallocate pre-allocate dest files on remote receiver\n");
+#endif
rprintf(F," -n, --dry-run perform a trial run with no changes made\n");
rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n");
rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
{"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 },
{"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 },
{"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 },
+ {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0},
{"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 },
{"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 },
{"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 },
else if (remove_source_files)
args[ac++] = "--remove-sent-files";
+ if (preallocate_files && am_sender)
+ args[ac++] = "--preallocate";
+
if (ac > MAX_SERVER_ARGS) { /* Not possible... */
rprintf(FERROR, "argc overflow in server_options().\n");
exit_cleanup(RERR_MALLOC);
extern int remove_source_files;
extern int append_mode;
extern int sparse_files;
+extern int preallocate_files;
extern int keep_partial;
extern int checksum_len;
extern int checksum_seed;
char *data;
int32 i;
char *map = NULL;
+#ifdef SUPPORT_PREALLOCATION
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ OFF_T preallocated_len = 0;
+#endif
+
+ if (preallocate_files && fd != -1 && total_size > 0) {
+ /* Try to preallocate enough space for file's eventual length. Can
+ * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
+ if (do_fallocate(fd, 0, total_size) == 0) {
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ preallocated_len = total_size;
+#endif
+ } else
+ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
+ }
+#endif
read_sum_head(f_in, &sum);
goto report_write_error;
#ifdef HAVE_FTRUNCATE
- if (inplace && fd != -1 && do_ftruncate(fd, offset) < 0) {
+ /* inplace: New data could be shorter than old data.
+ * preallocate_files: total_size could have been an overestimate.
+ * Cut off any extra preallocated zeros from dest file. */
+ if ((inplace
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ || preallocated_len > offset
+#endif
+ ) && fd != -1 && do_ftruncate(fd, offset) < 0) {
rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
full_fname(fname));
}
#define ACLS_NEED_MASK 1
#endif
+#if defined HAVE_FALLOCATE || HAVE_SYS_FALLOCATE
+#include <linux/falloc.h>
+#ifdef FALLOC_FL_KEEP_SIZE
+#define SUPPORT_PREALLOCATION 1
+#elif defined HAVE_FTRUNCATE
+#define SUPPORT_PREALLOCATION 1
+#define PREALLOCATE_NEEDS_TRUNCATE 1
+#endif
+#else /* !fallocate */
+#if defined HAVE_EFFICIENT_POSIX_FALLOCATE && defined HAVE_FTRUNCATE
+#define SUPPORT_PREALLOCATION 1
+#define PREALLOCATE_NEEDS_TRUNCATE 1
+#endif
+#endif
+
union file_extras {
int32 num;
uint32 unum;
--super receiver attempts super-user activities
--fake-super store/recover privileged attrs using xattrs
-S, --sparse handle sparse files efficiently
+ --preallocate allocate dest files before writing
-n, --dry-run perform a trial run with no changes made
-W, --whole-file copy files whole (w/o delta-xfer algorithm)
-x, --one-file-system don't cross filesystem boundaries
up less space on the destination. Conflicts with bf(--inplace) because it's
not possible to overwrite data in a sparse fashion.
+dit(bf(--preallocate)) This tells the receiver to allocate each destination
+file to its eventual size before writing data to the file. Rsync will only use
+the real filesystem-level preallocation support provided by Linux's
+bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow
+glibc implementation that writes a zero byte into each block.
+
+Without this option, larger files may not be entirely contiguous on the
+filesystem, but with this option rsync will probably copy more slowly. If the
+destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS,
+etc.), this option may have no positive effect at all.
+
dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't
make any changes (and produces mostly the same output as a real run). It
is most commonly used in combination with the bf(-v, --verbose) and/or
#include <sys/attr.h>
#endif
+#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE
+#include <sys/syscall.h>
+#endif
+
extern int dry_run;
extern int am_root;
extern int am_sender;
#else
#error Need utimes or utime function.
#endif
+
+#ifdef SUPPORT_PREALLOCATION
+int do_fallocate(int fd, OFF_T offset, OFF_T length)
+{
+#ifdef FALLOC_FL_KEEP_SIZE
+#define DO_FALLOC_OPTIONS FALLOC_FL_KEEP_SIZE
+#else
+#define DO_FALLOC_OPTIONS 0
+#endif
+ RETURN_ERROR_IF(dry_run, 0);
+ RETURN_ERROR_IF_RO_OR_LO;
+#if defined HAVE_FALLOCATE
+ return fallocate(fd, DO_FALLOC_OPTIONS, offset, length);
+#elif defined HAVE_SYS_FALLOCATE
+ return syscall(SYS_fallocate, fd, DO_FALLOC_OPTIONS, (loff_t)offset, (loff_t)length);
+#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE
+ return posix_fallocate(fd, offset, length);
+#else
+#error Coding error in SUPPORT_PREALLOCATION logic.
+#endif
+}
+#endif
#include "rsync.h"
int modify_window = 0;
+int preallocate_files = 0;
int module_id = -1;
int relative_paths = 0;
int module_dirlen = 0;
extern int relative_paths;
extern int preserve_times;
extern int preserve_xattrs;
+extern int preallocate_files;
extern char *module_dir;
extern unsigned int module_dirlen;
extern char *partial_dir;
int ifd;
char buf[1024 * 8];
int len; /* Number of bytes read into `buf'. */
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ OFF_T preallocated_len = 0, offset = 0;
+#endif
if ((ifd = do_open(source, O_RDONLY, 0)) < 0) {
int save_errno = errno;
}
}
+#ifdef SUPPORT_PREALLOCATION
+ if (preallocate_files) {
+ STRUCT_STAT srcst;
+
+ /* Try to preallocate enough space for file's eventual length. Can
+ * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
+ if (do_fstat(ifd, &srcst) < 0)
+ rsyserr(FWARNING, errno, "fstat %s", full_fname(source));
+ else if (srcst.st_size > 0) {
+ if (do_fallocate(ofd, 0, srcst.st_size) == 0) {
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ preallocated_len = srcst.st_size;
+#endif
+ } else
+ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest));
+ }
+ }
+#endif
+
while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
if (full_write(ofd, buf, len) < 0) {
int save_errno = errno;
errno = save_errno;
return -1;
}
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ offset += len;
+#endif
}
if (len < 0) {
full_fname(source));
}
+#ifdef PREALLOCATE_NEEDS_TRUNCATE
+ /* Source file might have shrunk since we fstatted it.
+ * Cut off any extra preallocated zeros from dest file. */
+ if (offset < preallocated_len && do_ftruncate(ofd, offset) < 0) {
+ /* If we fail to truncate, the dest file may be wrong, so we
+ * must trigger the "partial transfer" error. */
+ rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest));
+ }
+#endif
+
if (close(ofd) < 0) {
int save_errno = errno;
rsyserr(FERROR_XFER, errno, "close failed on %s",