1 This patch adds the --preallocate option that asks rsync to preallocate the
2 copied files. This slows down the copy, but should reduce fragmentation on
3 systems that need that.
5 To use this patch, run these commands for a successful build:
7 patch -p1 <patches/preallocate.diff
12 diff --git a/configure.in b/configure.in
15 @@ -552,7 +552,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \
16 strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
17 setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
18 strerror putenv iconv_open locale_charset nl_langinfo getxattr \
19 - extattr_get_link sigaction sigprocmask setattrlist)
20 + extattr_get_link sigaction sigprocmask setattrlist posix_fallocate)
22 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
23 if test $ac_cv_func_getpgrp = yes; then
24 diff --git a/options.c b/options.c
27 @@ -72,6 +72,7 @@ int remove_source_files = 0;
28 int one_file_system = 0;
29 int protocol_version = PROTOCOL_VERSION;
31 +int preallocate_files = 0;
32 int do_compression = 0;
33 int def_compress_level = Z_DEFAULT_COMPRESSION;
34 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
35 @@ -223,6 +224,7 @@ static void print_rsync_version(enum logcode f)
36 char const *links = "no ";
37 char const *iconv = "no ";
38 char const *ipv6 = "no ";
39 + char const *preallocation = "no ";
42 #if SUBPROTOCOL_VERSION != 0
43 @@ -252,6 +254,9 @@ static void print_rsync_version(enum logcode f)
47 +#ifdef SUPPORT_PREALLOCATION
51 rprintf(f, "%s version %s protocol version %d%s\n",
52 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
53 @@ -265,8 +270,8 @@ static void print_rsync_version(enum logcode f)
54 (int)(sizeof (int64) * 8));
55 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
56 got_socketpair, hardlinks, links, ipv6, have_inplace);
57 - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv\n",
58 - have_inplace, acls, xattrs, iconv);
59 + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %spreallocation\n",
60 + have_inplace, acls, xattrs, iconv, preallocation);
62 #ifdef MAINTAINER_MODE
63 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
64 @@ -353,6 +358,9 @@ void usage(enum logcode F)
65 rprintf(F," --fake-super store/recover privileged attrs using xattrs\n");
67 rprintf(F," -S, --sparse handle sparse files efficiently\n");
68 +#ifdef SUPPORT_PREALLOCATION
69 + rprintf(F," --preallocate posix_fallocate dest files before writing them\n");
71 rprintf(F," -n, --dry-run perform a trial run with no changes made\n");
72 rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n");
73 rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
74 @@ -531,6 +539,7 @@ static struct poptOption long_options[] = {
75 {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
76 {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
77 {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 },
78 + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0},
79 {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 },
80 {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 },
81 {"append-verify", 0, POPT_ARG_VAL, &append_mode, 2, 0, 0 },
82 @@ -1289,6 +1298,15 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain)
86 +#ifndef SUPPORT_PREALLOCATION
87 + if (preallocate_files && !am_sender) {
88 + snprintf(err_buf, sizeof err_buf,
89 + "preallocation is not supported on this %s\n",
90 + am_server ? "server" : "client");
95 if (write_batch && read_batch) {
96 snprintf(err_buf, sizeof err_buf,
97 "--write-batch and --read-batch can not be used together\n");
98 @@ -1999,6 +2017,9 @@ void server_options(char **args, int *argc_p)
99 else if (remove_source_files)
100 args[ac++] = "--remove-sent-files";
102 + if (preallocate_files && am_sender)
103 + args[ac++] = "--preallocate";
108 diff --git a/receiver.c b/receiver.c
111 @@ -45,6 +45,7 @@ extern int cleanup_got_literal;
112 extern int remove_source_files;
113 extern int append_mode;
114 extern int sparse_files;
115 +extern int preallocate_files;
116 extern int keep_partial;
117 extern int checksum_seed;
119 @@ -175,6 +176,19 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
123 +#ifdef SUPPORT_PREALLOCATION
124 + OFF_T preallocated_len = 0;
126 + if (preallocate_files && fd != -1 && total_size > 0) {
127 + /* Preallocate enough space for file's eventual length if
128 + * possible; seems to reduce fragmentation on Windows. */
129 + if (posix_fallocate(fd, 0, total_size) == 0)
130 + preallocated_len = total_size;
132 + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname));
136 read_sum_head(f_in, &sum);
138 if (fd_r >= 0 && size_r > 0) {
139 @@ -284,8 +298,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
140 goto report_write_error;
142 #ifdef HAVE_FTRUNCATE
143 - if (inplace && fd != -1)
144 - ftruncate(fd, offset);
145 + /* inplace: New data could be shorter than old data.
146 + * preallocate_files: total_size could have been an overestimate.
147 + * Cut off any extra preallocated zeros from dest file. */
149 +#ifdef SUPPORT_PREALLOCATION
150 + || preallocated_len > offset
153 + if (ftruncate(fd, offset) < 0)
154 + /* If we fail to truncate, the dest file may be wrong, so we
155 + * must trigger the "partial transfer" error. */
156 + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(fname));
160 diff --git a/rsync.h b/rsync.h
163 @@ -600,6 +600,10 @@ struct ht_int64_node {
164 #define ACLS_NEED_MASK 1
167 +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE
168 +#define SUPPORT_PREALLOCATION 1
174 diff --git a/rsync.yo b/rsync.yo
177 @@ -352,6 +352,7 @@ to the detailed description below for a complete description. verb(
178 --super receiver attempts super-user activities
179 --fake-super store/recover privileged attrs using xattrs
180 -S, --sparse handle sparse files efficiently
181 + --preallocate posix_fallocate dest files before writing
182 -n, --dry-run perform a trial run with no changes made
183 -W, --whole-file copy files whole (w/o delta-xfer algorithm)
184 -x, --one-file-system don't cross filesystem boundaries
185 @@ -1023,6 +1024,19 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs"
186 filesystem. It doesn't seem to handle seeks over null regions
187 correctly and ends up corrupting the files.
189 +dit(bf(--preallocate)) This tells the receiver to allocate each destination
190 +file to its eventual size using bf(posix_fallocate)(3) before writing data
191 +to the file. If the receiver is remote, this nonstandard option only works
192 +if the receiver also has the preallocation patch. Furthermore, this option
193 +only works if the receiver found the bf(posix_fallocate)(3) call at
196 +Without this option on MS Windows, very large destination files tend to be
197 +broken into thousands of fragments; advising Windows ahead of time of the
198 +eventual file size using this option usually reduces the number of
199 +fragments to one. However, on Linux, this option appears to just waste
202 dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't
203 make any changes (and produces mostly the same output as a real run). It
204 is most commonly used in combination with the bf(-v, --verbose) and/or
205 diff --git a/t_stub.c b/t_stub.c
211 int modify_window = 0;
212 +int preallocate_files = 0;
214 int relative_paths = 0;
215 int human_readable = 0;
216 diff --git a/util.c b/util.c
223 +extern int preallocate_files;
224 extern int module_id;
225 extern int modify_window;
226 extern int relative_paths;
227 @@ -272,6 +273,10 @@ int copy_file(const char *source, const char *dest, int ofd,
230 int len; /* Number of bytes read into `buf'. */
231 +#ifdef SUPPORT_PREALLOCATION
232 + OFF_T preallocated_len = 0;
236 if ((ifd = do_open(source, O_RDONLY, 0)) < 0) {
237 rsyserr(FERROR_XFER, errno, "open %s", full_fname(source));
238 @@ -293,7 +298,27 @@ int copy_file(const char *source, const char *dest, int ofd,
242 +#ifdef SUPPORT_PREALLOCATION
243 + if (preallocate_files) {
244 + /* Preallocate enough space for file's eventual length if
245 + * possible; seems to reduce fragmentation on Windows. */
247 + if (do_fstat(ifd, &srcst) == 0) {
248 + if (srcst.st_size > 0) {
249 + if (posix_fallocate(ofd, 0, srcst.st_size) == 0)
250 + preallocated_len = srcst.st_size;
252 + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest));
255 + rsyserr(FINFO, errno, "fstat %s", full_fname(source));
259 while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
260 +#ifdef SUPPORT_PREALLOCATION
263 if (full_write(ofd, buf, len) < 0) {
264 rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest));
266 @@ -314,6 +339,16 @@ int copy_file(const char *source, const char *dest, int ofd,
270 +#ifdef SUPPORT_PREALLOCATION
271 + /* Source file might have shrunk since we fstatted it.
272 + * Cut off any extra preallocated zeros from dest file. */
273 + if (preallocated_len > offset)
274 + if (ftruncate(ofd, offset) < 0)
275 + /* If we fail to truncate, the dest file may be wrong, so we
276 + * must trigger the "partial transfer" error. */
277 + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest));
280 if (close(ofd) < 0) {
281 rsyserr(FERROR_XFER, errno, "close failed on %s",