This patch adds the --preallocate option that asks rsync to preallocate the copied files. This slows down the copy, but should reduce fragmentation on systems that need that. To use this patch, run these commands for a successful build: patch -p1 0) { + /* Preallocate enough space for file's eventual length if + * possible; seems to reduce fragmentation on Windows. */ + if (posix_fallocate(fd, 0, total_size) == 0) + preallocated_len = total_size; + else + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); + } +#endif + read_sum_head(f_in, &sum); if (fd_r >= 0 && size_r > 0) { @@ -247,8 +261,18 @@ static int receive_data(int f_in, char * goto report_write_error; #ifdef HAVE_FTRUNCATE - if (inplace && fd != -1) - ftruncate(fd, offset); + /* inplace: New data could be shorter than old data. + * preallocate_files: total_size could have been an overestimate. + * Cut off any extra preallocated zeros from dest file. */ + if ((inplace +#ifdef SUPPORT_PREALLOCATION + || preallocated_len > offset +#endif + ) && fd != -1) + if (ftruncate(fd, offset) < 0) + /* If we fail to truncate, the dest file may be wrong, so we + * must trigger the "partial transfer" error. */ + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname)); #endif if (do_progress) --- old/rsync.h +++ new/rsync.h @@ -568,6 +568,10 @@ struct ht_int64_node { #define ACLS_NEED_MASK 1 #endif +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE +#define SUPPORT_PREALLOCATION 1 +#endif + union file_extras { int32 num; uint32 unum; --- old/rsync.yo +++ new/rsync.yo @@ -352,6 +352,7 @@ to the detailed description below for a --super receiver attempts super-user activities --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently + --preallocate posix_fallocate dest files before writing -n, --dry-run show what would have been transferred -W, --whole-file copy files whole (without rsync algorithm) -x, --one-file-system don't cross filesystem boundaries @@ -1014,6 +1015,19 @@ NOTE: Don't use this option when the des filesystem. It doesn't seem to handle seeks over null regions correctly and ends up corrupting the files. +dit(bf(--preallocate)) This tells the receiver to allocate each destination +file to its eventual size using bf(posix_fallocate)(3) before writing data +to the file. If the receiver is remote, this nonstandard option only works +if the receiver also has the preallocation patch. Furthermore, this option +only works if the receiver found the bf(posix_fallocate)(3) call at +configure time. + +Without this option on MS Windows, very large destination files tend to be +broken into thousands of fragments; advising Windows ahead of time of the +eventual file size using this option usually reduces the number of +fragments to one. However, on Linux, this option appears to just waste +disk I/O. + dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers, instead it will just report the actions it would have taken. --- old/t_stub.c +++ new/t_stub.c @@ -22,6 +22,7 @@ #include "rsync.h" int modify_window = 0; +int preallocate_files = 0; int module_id = -1; int relative_paths = 0; int human_readable = 0; --- old/util.c +++ new/util.c @@ -25,6 +25,7 @@ extern int verbose; extern int dry_run; +extern int preallocate_files; extern int module_id; extern int modify_window; extern int relative_paths; @@ -271,6 +272,10 @@ int copy_file(const char *source, const int ofd; char buf[1024 * 8]; int len; /* Number of bytes read into `buf'. */ +#ifdef SUPPORT_PREALLOCATION + int preallocated_len = 0; + int offset = 0; +#endif if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { rsyserr(FERROR, errno, "open %s", full_fname(source)); @@ -290,7 +295,27 @@ int copy_file(const char *source, const return -1; } +#ifdef SUPPORT_PREALLOCATION + if (preallocate_files) { + /* Preallocate enough space for file's eventual length if + * possible; seems to reduce fragmentation on Windows. */ + STRUCT_STAT srcst; + if (do_fstat(ifd, &srcst) == 0) { + if (srcst.st_size > 0) { + if (posix_fallocate(ofd, 0, srcst.st_size) == 0) + preallocated_len = srcst.st_size; + else + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); + } + } else + rsyserr(FINFO, errno, "fstat %s", full_fname(source)); + } +#endif + while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { +#ifdef SUPPORT_PREALLOCATION + offset += len; +#endif if (full_write(ofd, buf, len) < 0) { rsyserr(FERROR, errno, "write %s", full_fname(dest)); close(ifd); @@ -311,6 +336,16 @@ int copy_file(const char *source, const full_fname(source)); } +#ifdef SUPPORT_PREALLOCATION + /* Source file might have shrunk since we fstatted it. + * Cut off any extra preallocated zeros from dest file. */ + if (preallocated_len > offset) + if (ftruncate(ofd, offset) < 0) + /* If we fail to truncate, the dest file may be wrong, so we + * must trigger the "partial transfer" error. */ + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest)); +#endif + if (close(ofd) < 0) { rsyserr(FERROR, errno, "close failed on %s", full_fname(dest));