1 This patch adds the --preallocate option that asks rsync to preallocate the
2 copied files. This slows down the copy, but should reduce fragmentation on
3 systems that need that.
5 To use this patch, run these commands for a successful build:
7 patch -p1 <patches/preallocate.diff
14 @@ -560,7 +560,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd
15 strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
16 setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
17 strerror putenv iconv_open locale_charset nl_langinfo \
18 - sigaction sigprocmask)
19 + sigaction sigprocmask posix_fallocate)
21 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
22 if test $ac_cv_func_getpgrp = yes; then
25 @@ -69,6 +69,7 @@ int remove_source_files = 0;
26 int one_file_system = 0;
27 int protocol_version = PROTOCOL_VERSION;
29 +int preallocate_files = 0;
30 int do_compression = 0;
31 int def_compress_level = Z_DEFAULT_COMPRESSION;
33 @@ -202,6 +203,7 @@ static void print_rsync_version(enum log
34 char const *acls = "no ";
35 char const *links = "no ";
36 char const *ipv6 = "no ";
37 + char const *preallocation = "no ";
40 #ifdef HAVE_SOCKETPAIR
41 @@ -228,6 +230,10 @@ static void print_rsync_version(enum log
45 +#ifdef SUPPORT_PREALLOCATION
49 rprintf(f, "%s version %s protocol version %d\n",
50 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION);
51 rprintf(f, "Copyright (C) 1996-2007 by Andrew Tridgell, Wayne Davison, and others.\n");
52 @@ -238,8 +244,8 @@ static void print_rsync_version(enum log
53 (int)(sizeof (int64) * 8));
54 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
55 got_socketpair, hardlinks, links, ipv6, have_inplace);
56 - rprintf(f, " %sappend, %sACLs\n",
57 - have_inplace, acls);
58 + rprintf(f, " %sappend, %sACLs, %spreallocation\n",
59 + have_inplace, acls, preallocation);
61 #ifdef MAINTAINER_MODE
62 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
63 @@ -319,6 +325,9 @@ void usage(enum logcode F)
64 rprintf(F," -O, --omit-dir-times omit directories when preserving times\n");
65 rprintf(F," --super receiver attempts super-user activities\n");
66 rprintf(F," -S, --sparse handle sparse files efficiently\n");
67 +#ifdef SUPPORT_PREALLOCATION
68 + rprintf(F," --preallocate posix_fallocate dest files before writing them\n");
70 rprintf(F," -n, --dry-run show what would have been transferred\n");
71 rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n");
72 rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
73 @@ -479,6 +488,7 @@ static struct poptOption long_options[]
74 {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 },
75 {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 },
76 {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 },
77 + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0},
78 {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 },
79 {"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 },
80 {"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 },
81 @@ -1161,6 +1171,15 @@ int parse_arguments(int *argc, const cha
85 +#ifndef SUPPORT_PREALLOCATION
86 + if (preallocate_files && !am_sender) {
87 + snprintf(err_buf, sizeof err_buf,
88 + "preallocation is not supported on this %s\n",
89 + am_server ? "server" : "client");
94 if (write_batch && read_batch) {
95 snprintf(err_buf, sizeof err_buf,
96 "--write-batch and --read-batch can not be used together\n");
97 @@ -1808,6 +1827,9 @@ void server_options(char **args,int *arg
98 else if (remove_source_files)
99 args[ac++] = "--remove-sent-files";
101 + if (preallocate_files && am_sender)
102 + args[ac++] = "--preallocate";
109 @@ -43,6 +43,7 @@ extern int cleanup_got_literal;
110 extern int remove_source_files;
111 extern int append_mode;
112 extern int sparse_files;
113 +extern int preallocate_files;
114 extern int keep_partial;
115 extern int checksum_seed;
117 @@ -136,6 +137,19 @@ static int receive_data(int f_in, char *
121 +#ifdef SUPPORT_PREALLOCATION
122 + int preallocated_len = 0;
124 + if (preallocate_files && fd != -1 && total_size > 0) {
125 + /* Preallocate enough space for file's eventual length if
126 + * possible; seems to reduce fragmentation on Windows. */
127 + if (posix_fallocate(fd, 0, total_size) == 0)
128 + preallocated_len = total_size;
130 + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname));
134 read_sum_head(f_in, &sum);
136 if (fd_r >= 0 && size_r > 0) {
137 @@ -243,8 +257,18 @@ static int receive_data(int f_in, char *
138 goto report_write_error;
140 #ifdef HAVE_FTRUNCATE
141 - if (inplace && fd != -1)
142 - ftruncate(fd, offset);
143 + /* inplace: New data could be shorter than old data.
144 + * preallocate_files: total_size could have been an overestimate.
145 + * Cut off any extra preallocated zeros from dest file. */
147 +#ifdef SUPPORT_PREALLOCATION
148 + || preallocated_len > offset
151 + if (ftruncate(fd, offset) < 0)
152 + /* If we fail to truncate, the dest file may be wrong, so we
153 + * must trigger the "partial transfer" error. */
154 + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname));
160 @@ -551,6 +551,10 @@ struct idev_node {
161 #define ACLS_NEED_MASK 1
164 +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE
165 +#define SUPPORT_PREALLOCATION 1
168 #define GID_NONE ((gid_t)-1)
173 @@ -333,6 +333,7 @@ to the detailed description below for a
174 -O, --omit-dir-times omit directories when preserving times
175 --super receiver attempts super-user activities
176 -S, --sparse handle sparse files efficiently
177 + --preallocate posix_fallocate dest files before writing
178 -n, --dry-run show what would have been transferred
179 -W, --whole-file copy files whole (without rsync algorithm)
180 -x, --one-file-system don't cross filesystem boundaries
181 @@ -914,6 +915,19 @@ NOTE: Don't use this option when the des
182 filesystem. It doesn't seem to handle seeks over null regions
183 correctly and ends up corrupting the files.
185 +dit(bf(--preallocate)) This tells the receiver to allocate each destination
186 +file to its eventual size using bf(posix_fallocate)(3) before writing data
187 +to the file. If the receiver is remote, this nonstandard option only works
188 +if the receiver also has the preallocation patch. Furthermore, this option
189 +only works if the receiver found the bf(posix_fallocate)(3) call at
192 +Without this option on MS Windows, very large destination files tend to be
193 +broken into thousands of fragments; advising Windows ahead of time of the
194 +eventual file size using this option usually reduces the number of
195 +fragments to one. However, on Linux, this option appears to just waste
198 dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers,
199 instead it will just report the actions it would have taken.
206 int modify_window = 0;
207 +int preallocate_files = 0;
209 int relative_paths = 0;
210 int human_readable = 0;
217 +extern int preallocate_files;
218 extern int module_id;
219 extern int modify_window;
220 extern int relative_paths;
221 @@ -269,6 +270,10 @@ int copy_file(const char *source, const
224 int len; /* Number of bytes read into `buf'. */
225 +#ifdef SUPPORT_PREALLOCATION
226 + int preallocated_len = 0;
230 ifd = do_open(source, O_RDONLY, 0);
232 @@ -288,7 +293,27 @@ int copy_file(const char *source, const
236 +#ifdef SUPPORT_PREALLOCATION
237 + if (preallocate_files) {
238 + /* Preallocate enough space for file's eventual length if
239 + * possible; seems to reduce fragmentation on Windows. */
241 + if (do_fstat(ifd, &srcst) == 0) {
242 + if (srcst.st_size > 0) {
243 + if (posix_fallocate(ofd, 0, srcst.st_size) == 0)
244 + preallocated_len = srcst.st_size;
246 + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest));
249 + rsyserr(FINFO, errno, "fstat %s", full_fname(source));
253 while ((len = safe_read(ifd, buf, sizeof buf)) > 0) {
254 +#ifdef SUPPORT_PREALLOCATION
257 if (full_write(ofd, buf, len) < 0) {
258 rsyserr(FERROR, errno, "write %s", full_fname(dest));
260 @@ -309,6 +334,16 @@ int copy_file(const char *source, const
264 +#ifdef SUPPORT_PREALLOCATION
265 + /* Source file might have shrunk since we fstatted it.
266 + * Cut off any extra preallocated zeros from dest file. */
267 + if (preallocated_len > offset)
268 + if (ftruncate(ofd, offset) < 0)
269 + /* If we fail to truncate, the dest file may be wrong, so we
270 + * must trigger the "partial transfer" error. */
271 + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest));
274 if (close(ofd) < 0) {
275 rsyserr(FERROR, errno, "close failed on %s",