Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
f9df736a WD |
12 | diff --git a/compat.c b/compat.c |
13 | --- a/compat.c | |
14 | +++ b/compat.c | |
15 | @@ -33,6 +33,7 @@ extern int inplace; | |
16 | extern int recurse; | |
17 | extern int use_qsort; | |
18 | extern int allow_inc_recurse; | |
19 | +extern int preallocate_files; | |
20 | extern int append_mode; | |
21 | extern int fuzzy_basis; | |
22 | extern int read_batch; | |
23 | @@ -184,6 +185,15 @@ void setup_protocol(int f_out,int f_in) | |
24 | if (read_batch) | |
25 | check_batch_flags(); | |
26 | ||
27 | +#ifndef SUPPORT_PREALLOCATION | |
28 | + if (preallocate_files && !am_sender) { | |
29 | + rprintf(FERROR, | |
30 | + "preallocation is not supported on this %s\n", | |
31 | + am_server ? "server" : "client"); | |
32 | + exit_cleanup(RERR_SYNTAX); | |
33 | + } | |
34 | +#endif | |
35 | + | |
36 | if (protocol_version < 30) { | |
37 | if (append_mode == 1) | |
38 | append_mode = 2; | |
cc3e685d WD |
39 | diff --git a/configure.in b/configure.in |
40 | --- a/configure.in | |
41 | +++ b/configure.in | |
c0c7984e | 42 | @@ -554,13 +554,40 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ |
5e3c6c93 WD |
43 | strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ |
44 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ | |
80c89075 | 45 | strerror putenv iconv_open locale_charset nl_langinfo getxattr \ |
7c4c2959 | 46 | - extattr_get_link sigaction sigprocmask setattrlist) |
f9df736a | 47 | + extattr_get_link sigaction sigprocmask setattrlist fallocate posix_fallocate) |
5e3c6c93 | 48 | |
4c15e800 WD |
49 | dnl cygwin iconv.h defines iconv_open as libiconv_open |
50 | if test x"$ac_cv_func_iconv_open" != x"yes"; then | |
f9df736a WD |
51 | AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) |
52 | fi | |
53 | ||
54 | +dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): | |
55 | + | |
56 | +AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ | |
57 | +AC_TRY_COMPILE([#include <sys/syscall.h> | |
58 | +#include <sys/types.h>], | |
59 | +[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);], | |
60 | +rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) | |
61 | +if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then | |
62 | + AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) | |
63 | +fi | |
64 | + | |
65 | +if test x"$ac_cv_func_posix_fallocate" = x"yes"; then | |
66 | + AC_MSG_CHECKING([whether posix_fallocate is efficient]) | |
67 | + case $host_os in | |
68 | + *cygwin*) | |
69 | + AC_MSG_RESULT(yes) | |
70 | + AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, | |
71 | + [Define if posix_fallocate is efficient (Cygwin)]) | |
72 | + ;; | |
73 | + *) | |
74 | + AC_MSG_RESULT(no) | |
75 | + ;; | |
76 | + esac | |
77 | +fi | |
78 | + | |
79 | +dnl End of preallocation stuff | |
80 | + | |
81 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) | |
82 | if test $ac_cv_func_getpgrp = yes; then | |
83 | AC_FUNC_GETPGRP | |
cc3e685d WD |
84 | diff --git a/options.c b/options.c |
85 | --- a/options.c | |
86 | +++ b/options.c | |
c0c7984e | 87 | @@ -73,6 +73,7 @@ int remove_source_files = 0; |
5e3c6c93 WD |
88 | int one_file_system = 0; |
89 | int protocol_version = PROTOCOL_VERSION; | |
90 | int sparse_files = 0; | |
91 | +int preallocate_files = 0; | |
92 | int do_compression = 0; | |
93 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
58b399b9 | 94 | int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ |
c0c7984e | 95 | @@ -225,6 +226,7 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 | 96 | char const *links = "no "; |
58b399b9 | 97 | char const *iconv = "no "; |
5e3c6c93 WD |
98 | char const *ipv6 = "no "; |
99 | + char const *preallocation = "no "; | |
100 | STRUCT_STAT *dumstat; | |
101 | ||
ac2da598 | 102 | #if SUBPROTOCOL_VERSION != 0 |
c0c7984e | 103 | @@ -257,6 +259,9 @@ static void print_rsync_version(enum logcode f) |
85096e5e WD |
104 | #if defined HAVE_LUTIMES && defined HAVE_UTIMES |
105 | symtimes = ""; | |
5e3c6c93 | 106 | #endif |
5e3c6c93 WD |
107 | +#ifdef SUPPORT_PREALLOCATION |
108 | + preallocation = ""; | |
109 | +#endif | |
ac2da598 WD |
110 | |
111 | rprintf(f, "%s version %s protocol version %d%s\n", | |
112 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
c0c7984e | 113 | @@ -270,8 +275,8 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 WD |
114 | (int)(sizeof (int64) * 8)); |
115 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
116 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
85096e5e WD |
117 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", |
118 | - have_inplace, acls, xattrs, iconv, symtimes); | |
119 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %spreallocation\n", | |
120 | + have_inplace, acls, xattrs, iconv, symtimes, preallocation); | |
5e3c6c93 WD |
121 | |
122 | #ifdef MAINTAINER_MODE | |
123 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
c0c7984e | 124 | @@ -358,6 +363,9 @@ void usage(enum logcode F) |
58b399b9 WD |
125 | rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); |
126 | #endif | |
5e3c6c93 WD |
127 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
128 | +#ifdef SUPPORT_PREALLOCATION | |
f9df736a | 129 | + rprintf(F," --preallocate allocate dest files before writing them\n"); |
5e3c6c93 | 130 | +#endif |
e2b0842a | 131 | rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); |
f2863bc0 | 132 | rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); |
5e3c6c93 | 133 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); |
c0c7984e WD |
134 | @@ -542,6 +550,7 @@ static struct poptOption long_options[] = { |
135 | {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, | |
136 | {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
137 | {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
5e3c6c93 | 138 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, |
c0c7984e WD |
139 | {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, |
140 | {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, | |
790ba11a | 141 | {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, |
c0c7984e | 142 | @@ -2048,6 +2057,9 @@ void server_options(char **args, int *argc_p) |
5e3c6c93 WD |
143 | else if (remove_source_files) |
144 | args[ac++] = "--remove-sent-files"; | |
145 | ||
146 | + if (preallocate_files && am_sender) | |
147 | + args[ac++] = "--preallocate"; | |
148 | + | |
790ba11a | 149 | *argc_p = ac; |
5e3c6c93 WD |
150 | return; |
151 | ||
cc3e685d WD |
152 | diff --git a/receiver.c b/receiver.c |
153 | --- a/receiver.c | |
154 | +++ b/receiver.c | |
5795bf59 | 155 | @@ -45,6 +45,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
156 | extern int remove_source_files; |
157 | extern int append_mode; | |
158 | extern int sparse_files; | |
159 | +extern int preallocate_files; | |
160 | extern int keep_partial; | |
161 | extern int checksum_seed; | |
162 | extern int inplace; | |
f9df736a WD |
163 | @@ -174,6 +175,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
164 | char *data; | |
5e3c6c93 WD |
165 | int32 i; |
166 | char *map = NULL; | |
5e3c6c93 | 167 | +#ifdef SUPPORT_PREALLOCATION |
cc3e685d | 168 | + OFF_T preallocated_len = 0; |
5e3c6c93 WD |
169 | + |
170 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
171 | + /* Preallocate enough space for file's eventual length if | |
172 | + * possible; seems to reduce fragmentation on Windows. */ | |
f9df736a | 173 | + if (do_fallocate(fd, 0, total_size) == 0) |
5e3c6c93 WD |
174 | + preallocated_len = total_size; |
175 | + else | |
f9df736a | 176 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); |
5e3c6c93 WD |
177 | + } |
178 | +#endif | |
f9df736a | 179 | |
5e3c6c93 WD |
180 | read_sum_head(f_in, &sum); |
181 | ||
f9df736a | 182 | @@ -284,8 +297,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
5e3c6c93 WD |
183 | goto report_write_error; |
184 | ||
185 | #ifdef HAVE_FTRUNCATE | |
186 | - if (inplace && fd != -1) | |
187 | - ftruncate(fd, offset); | |
188 | + /* inplace: New data could be shorter than old data. | |
189 | + * preallocate_files: total_size could have been an overestimate. | |
190 | + * Cut off any extra preallocated zeros from dest file. */ | |
191 | + if ((inplace | |
192 | +#ifdef SUPPORT_PREALLOCATION | |
193 | + || preallocated_len > offset | |
194 | +#endif | |
195 | + ) && fd != -1) | |
196 | + if (ftruncate(fd, offset) < 0) | |
197 | + /* If we fail to truncate, the dest file may be wrong, so we | |
198 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 199 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(fname)); |
5e3c6c93 WD |
200 | #endif |
201 | ||
202 | if (do_progress) | |
cc3e685d WD |
203 | diff --git a/rsync.h b/rsync.h |
204 | --- a/rsync.h | |
205 | +++ b/rsync.h | |
e8972101 | 206 | @@ -612,6 +612,13 @@ struct ht_int64_node { |
ffc18846 | 207 | #define ACLS_NEED_MASK 1 |
5e3c6c93 WD |
208 | #endif |
209 | ||
f9df736a WD |
210 | +#if defined HAVE_FTRUNCATE \ |
211 | + && (defined HAVE_FALLOCATE \ | |
212 | + || defined HAVE_SYS_FALLOCATE \ | |
213 | + || defined HAVE_EFFICIENT_POSIX_FALLOCATE) | |
5e3c6c93 WD |
214 | +#define SUPPORT_PREALLOCATION 1 |
215 | +#endif | |
216 | + | |
612d3765 | 217 | union file_extras { |
c4bd76ea WD |
218 | int32 num; |
219 | uint32 unum; | |
cc3e685d WD |
220 | diff --git a/rsync.yo b/rsync.yo |
221 | --- a/rsync.yo | |
222 | +++ b/rsync.yo | |
223 | @@ -352,6 +352,7 @@ to the detailed description below for a complete description. verb( | |
612d3765 | 224 | --super receiver attempts super-user activities |
58b399b9 | 225 | --fake-super store/recover privileged attrs using xattrs |
612d3765 | 226 | -S, --sparse handle sparse files efficiently |
f9df736a | 227 | + --preallocate allocate dest files before writing |
e2b0842a | 228 | -n, --dry-run perform a trial run with no changes made |
f2863bc0 | 229 | -W, --whole-file copy files whole (w/o delta-xfer algorithm) |
612d3765 | 230 | -x, --one-file-system don't cross filesystem boundaries |
91270139 | 231 | @@ -1049,6 +1050,18 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" |
612d3765 WD |
232 | filesystem. It doesn't seem to handle seeks over null regions |
233 | correctly and ends up corrupting the files. | |
234 | ||
235 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
f9df736a WD |
236 | +file to its eventual size before writing data to the file. Rsync will only use |
237 | +the real filesystem-level preallocation support provided by bf(fallocate)(2) or | |
238 | +Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes | |
239 | +a zero byte into each block. If the receiver is remote, this nonstandard | |
240 | +option only works if the receiver also has the preallocation patch. | |
612d3765 WD |
241 | + |
242 | +Without this option on MS Windows, very large destination files tend to be | |
243 | +broken into thousands of fragments; advising Windows ahead of time of the | |
244 | +eventual file size using this option usually reduces the number of | |
f9df736a | 245 | +fragments to one. The usefulness of this option on Linux is yet to be tested. |
612d3765 | 246 | + |
e2b0842a WD |
247 | dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't |
248 | make any changes (and produces mostly the same output as a real run). It | |
249 | is most commonly used in combination with the bf(-v, --verbose) and/or | |
f9df736a WD |
250 | diff --git a/syscall.c b/syscall.c |
251 | --- a/syscall.c | |
252 | +++ b/syscall.c | |
253 | @@ -29,6 +29,10 @@ | |
254 | #include <sys/attr.h> | |
255 | #endif | |
256 | ||
257 | +#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE | |
258 | +#include <sys/syscall.h> | |
259 | +#endif | |
260 | + | |
261 | extern int dry_run; | |
262 | extern int am_root; | |
263 | extern int read_only; | |
c0c7984e WD |
264 | @@ -282,3 +286,21 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence) |
265 | return lseek(fd, offset, whence); | |
f9df736a WD |
266 | #endif |
267 | } | |
268 | + | |
269 | +#ifdef SUPPORT_PREALLOCATION | |
270 | +int do_fallocate(int fd, OFF_T offset, OFF_T length) | |
271 | +{ | |
272 | + RETURN_ERROR_IF(dry_run, 0); | |
273 | + RETURN_ERROR_IF_RO_OR_LO; | |
274 | + /* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */ | |
275 | +#if defined HAVE_FALLOCATE | |
276 | + return fallocate(fd, 0, offset, length); | |
277 | +#elif defined HAVE_SYS_FALLOCATE | |
278 | + return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length); | |
279 | +#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE | |
280 | + return posix_fallocate(fd, offset, length); | |
281 | +#else | |
282 | +#error coding error in SUPPORT_PREALLOCATION | |
283 | +#endif | |
284 | +} | |
285 | +#endif | |
cc3e685d WD |
286 | diff --git a/t_stub.c b/t_stub.c |
287 | --- a/t_stub.c | |
288 | +++ b/t_stub.c | |
ffc18846 | 289 | @@ -22,6 +22,7 @@ |
5e3c6c93 WD |
290 | #include "rsync.h" |
291 | ||
292 | int modify_window = 0; | |
293 | +int preallocate_files = 0; | |
294 | int module_id = -1; | |
295 | int relative_paths = 0; | |
296 | int human_readable = 0; | |
cc3e685d WD |
297 | diff --git a/util.c b/util.c |
298 | --- a/util.c | |
299 | +++ b/util.c | |
c8a8b4a7 | 300 | @@ -25,6 +25,7 @@ |
5e3c6c93 WD |
301 | |
302 | extern int verbose; | |
303 | extern int dry_run; | |
304 | +extern int preallocate_files; | |
305 | extern int module_id; | |
306 | extern int modify_window; | |
307 | extern int relative_paths; | |
91270139 | 308 | @@ -276,6 +277,10 @@ int copy_file(const char *source, const char *dest, int ofd, |
e2b0842a | 309 | int ifd; |
5e3c6c93 WD |
310 | char buf[1024 * 8]; |
311 | int len; /* Number of bytes read into `buf'. */ | |
312 | +#ifdef SUPPORT_PREALLOCATION | |
cc3e685d WD |
313 | + OFF_T preallocated_len = 0; |
314 | + OFF_T offset = 0; | |
5e3c6c93 WD |
315 | +#endif |
316 | ||
c8a8b4a7 | 317 | if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { |
91270139 WD |
318 | int save_errno = errno; |
319 | @@ -309,7 +314,27 @@ int copy_file(const char *source, const char *dest, int ofd, | |
e2b0842a | 320 | } |
5e3c6c93 WD |
321 | } |
322 | ||
323 | +#ifdef SUPPORT_PREALLOCATION | |
324 | + if (preallocate_files) { | |
325 | + /* Preallocate enough space for file's eventual length if | |
326 | + * possible; seems to reduce fragmentation on Windows. */ | |
327 | + STRUCT_STAT srcst; | |
328 | + if (do_fstat(ifd, &srcst) == 0) { | |
329 | + if (srcst.st_size > 0) { | |
f9df736a | 330 | + if (do_fallocate(ofd, 0, srcst.st_size) == 0) |
5e3c6c93 WD |
331 | + preallocated_len = srcst.st_size; |
332 | + else | |
f9df736a | 333 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); |
5e3c6c93 WD |
334 | + } |
335 | + } else | |
f9df736a | 336 | + rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); |
5e3c6c93 WD |
337 | + } |
338 | +#endif | |
339 | + | |
340 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
341 | +#ifdef SUPPORT_PREALLOCATION | |
342 | + offset += len; | |
343 | +#endif | |
344 | if (full_write(ofd, buf, len) < 0) { | |
91270139 | 345 | int save_errno = errno; |
cc3e685d | 346 | rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest)); |
91270139 | 347 | @@ -334,6 +359,16 @@ int copy_file(const char *source, const char *dest, int ofd, |
5e3c6c93 WD |
348 | full_fname(source)); |
349 | } | |
350 | ||
351 | +#ifdef SUPPORT_PREALLOCATION | |
352 | + /* Source file might have shrunk since we fstatted it. | |
353 | + * Cut off any extra preallocated zeros from dest file. */ | |
354 | + if (preallocated_len > offset) | |
355 | + if (ftruncate(ofd, offset) < 0) | |
356 | + /* If we fail to truncate, the dest file may be wrong, so we | |
357 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 358 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); |
5e3c6c93 WD |
359 | +#endif |
360 | + | |
361 | if (close(ofd) < 0) { | |
91270139 | 362 | int save_errno = errno; |
cc3e685d | 363 | rsyserr(FERROR_XFER, errno, "close failed on %s", |