1 This patch adds the --preallocate option that asks rsync to preallocate the
2 copied files. This slows down the copy, but should reduce fragmentation on
3 systems that need that.
5 To use this patch, run these commands for a successful build:
7 patch -p1 <patches/preallocate.diff
14 @@ -561,7 +561,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd
15 strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
16 setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
17 strerror putenv iconv_open locale_charset nl_langinfo getxattr \
18 - extattr_get_link sigaction sigprocmask)
19 + extattr_get_link sigaction sigprocmask posix_fallocate)
21 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
22 if test $ac_cv_func_getpgrp = yes; then
25 @@ -70,6 +70,7 @@ int remove_source_files = 0;
26 int one_file_system = 0;
27 int protocol_version = PROTOCOL_VERSION;
29 +int preallocate_files = 0;
30 int do_compression = 0;
31 int def_compress_level = Z_DEFAULT_COMPRESSION;
32 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
33 @@ -212,6 +213,7 @@ static void print_rsync_version(enum log
34 char const *links = "no ";
35 char const *iconv = "no ";
36 char const *ipv6 = "no ";
37 + char const *preallocation = "no ";
40 #if SUBPROTOCOL_VERSION != 0
41 @@ -241,6 +243,9 @@ static void print_rsync_version(enum log
45 +#ifdef SUPPORT_PREALLOCATION
49 rprintf(f, "%s version %s protocol version %d%s\n",
50 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
51 @@ -254,8 +259,8 @@ static void print_rsync_version(enum log
52 (int)(sizeof (int64) * 8));
53 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
54 got_socketpair, hardlinks, links, ipv6, have_inplace);
55 - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv\n",
56 - have_inplace, acls, xattrs, iconv);
57 + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %spreallocation\n",
58 + have_inplace, acls, xattrs, iconv, preallocation);
60 #ifdef MAINTAINER_MODE
61 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
62 @@ -341,6 +346,9 @@ void usage(enum logcode F)
63 rprintf(F," --fake-super store/recover privileged attrs using xattrs\n");
65 rprintf(F," -S, --sparse handle sparse files efficiently\n");
66 +#ifdef SUPPORT_PREALLOCATION
67 + rprintf(F," --preallocate posix_fallocate dest files before writing them\n");
69 rprintf(F," -n, --dry-run show what would have been transferred\n");
70 rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n");
71 rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
72 @@ -512,6 +520,7 @@ static struct poptOption long_options[]
73 {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
74 {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
75 {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 },
76 + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0},
77 {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 },
78 {"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 },
79 {"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 },
80 @@ -1236,6 +1245,15 @@ int parse_arguments(int *argc, const cha
84 +#ifndef SUPPORT_PREALLOCATION
85 + if (preallocate_files && !am_sender) {
86 + snprintf(err_buf, sizeof err_buf,
87 + "preallocation is not supported on this %s\n",
88 + am_server ? "server" : "client");
93 if (write_batch && read_batch) {
94 snprintf(err_buf, sizeof err_buf,
95 "--write-batch and --read-batch can not be used together\n");
96 @@ -1911,6 +1929,9 @@ void server_options(char **args,int *arg
97 else if (remove_source_files)
98 args[ac++] = "--remove-sent-files";
100 + if (preallocate_files && am_sender)
101 + args[ac++] = "--preallocate";
108 @@ -45,6 +45,7 @@ extern int cleanup_got_literal;
109 extern int remove_source_files;
110 extern int append_mode;
111 extern int sparse_files;
112 +extern int preallocate_files;
113 extern int keep_partial;
114 extern int checksum_seed;
116 @@ -138,6 +139,19 @@ static int receive_data(int f_in, char *
120 +#ifdef SUPPORT_PREALLOCATION
121 + int preallocated_len = 0;
123 + if (preallocate_files && fd != -1 && total_size > 0) {
124 + /* Preallocate enough space for file's eventual length if
125 + * possible; seems to reduce fragmentation on Windows. */
126 + if (posix_fallocate(fd, 0, total_size) == 0)
127 + preallocated_len = total_size;
129 + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname));
133 read_sum_head(f_in, &sum);
135 if (fd_r >= 0 && size_r > 0) {
136 @@ -245,8 +259,18 @@ static int receive_data(int f_in, char *
137 goto report_write_error;
139 #ifdef HAVE_FTRUNCATE
140 - if (inplace && fd != -1)
141 - ftruncate(fd, offset);
142 + /* inplace: New data could be shorter than old data.
143 + * preallocate_files: total_size could have been an overestimate.
144 + * Cut off any extra preallocated zeros from dest file. */
146 +#ifdef SUPPORT_PREALLOCATION
147 + || preallocated_len > offset
150 + if (ftruncate(fd, offset) < 0)
151 + /* If we fail to truncate, the dest file may be wrong, so we
152 + * must trigger the "partial transfer" error. */
153 + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname));
159 @@ -565,6 +565,10 @@ struct idev_node {
160 #define ACLS_NEED_MASK 1
163 +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE
164 +#define SUPPORT_PREALLOCATION 1
172 @@ -341,6 +341,7 @@ to the detailed description below for a
173 --super receiver attempts super-user activities
174 --fake-super store/recover privileged attrs using xattrs
175 -S, --sparse handle sparse files efficiently
176 + --preallocate posix_fallocate dest files before writing
177 -n, --dry-run show what would have been transferred
178 -W, --whole-file copy files whole (without rsync algorithm)
179 -x, --one-file-system don't cross filesystem boundaries
180 @@ -962,6 +963,19 @@ NOTE: Don't use this option when the des
181 filesystem. It doesn't seem to handle seeks over null regions
182 correctly and ends up corrupting the files.
184 +dit(bf(--preallocate)) This tells the receiver to allocate each destination
185 +file to its eventual size using bf(posix_fallocate)(3) before writing data
186 +to the file. If the receiver is remote, this nonstandard option only works
187 +if the receiver also has the preallocation patch. Furthermore, this option
188 +only works if the receiver found the bf(posix_fallocate)(3) call at
191 +Without this option on MS Windows, very large destination files tend to be
192 +broken into thousands of fragments; advising Windows ahead of time of the
193 +eventual file size using this option usually reduces the number of
194 +fragments to one. However, on Linux, this option appears to just waste
197 dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers,
198 instead it will just report the actions it would have taken.
205 int modify_window = 0;
206 +int preallocate_files = 0;
208 int relative_paths = 0;
209 int human_readable = 0;
216 +extern int preallocate_files;
217 extern int module_id;
218 extern int modify_window;
219 extern int relative_paths;
220 @@ -269,6 +270,10 @@ int copy_file(const char *source, const
223 int len; /* Number of bytes read into `buf'. */
224 +#ifdef SUPPORT_PREALLOCATION
225 + int preallocated_len = 0;
229 ifd = do_open(source, O_RDONLY, 0);
231 @@ -288,7 +293,27 @@ int copy_file(const char *source, const
235 +#ifdef SUPPORT_PREALLOCATION
236 + if (preallocate_files) {
237 + /* Preallocate enough space for file's eventual length if
238 + * possible; seems to reduce fragmentation on Windows. */
240 + if (do_fstat(ifd, &srcst) == 0) {
241 + if (srcst.st_size > 0) {
242 + if (posix_fallocate(ofd, 0, srcst.st_size) == 0)
243 + preallocated_len = srcst.st_size;
245 + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest));
248 + rsyserr(FINFO, errno, "fstat %s", full_fname(source));
252 while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
253 +#ifdef SUPPORT_PREALLOCATION
256 if (full_write(ofd, buf, len) < 0) {
257 rsyserr(FERROR, errno, "write %s", full_fname(dest));
259 @@ -309,6 +334,16 @@ int copy_file(const char *source, const
263 +#ifdef SUPPORT_PREALLOCATION
264 + /* Source file might have shrunk since we fstatted it.
265 + * Cut off any extra preallocated zeros from dest file. */
266 + if (preallocated_len > offset)
267 + if (ftruncate(ofd, offset) < 0)
268 + /* If we fail to truncate, the dest file may be wrong, so we
269 + * must trigger the "partial transfer" error. */
270 + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest));
273 if (close(ofd) < 0) {
274 rsyserr(FERROR, errno, "close failed on %s",