Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
f9df736a | 12 | diff --git a/compat.c b/compat.c |
fc557362 | 13 | index 6e00072..c9590cc 100644 |
f9df736a WD |
14 | --- a/compat.c |
15 | +++ b/compat.c | |
fc557362 | 16 | @@ -32,6 +32,7 @@ extern int inplace; |
f9df736a WD |
17 | extern int recurse; |
18 | extern int use_qsort; | |
19 | extern int allow_inc_recurse; | |
20 | +extern int preallocate_files; | |
21 | extern int append_mode; | |
22 | extern int fuzzy_basis; | |
23 | extern int read_batch; | |
fc557362 | 24 | @@ -186,6 +187,15 @@ void setup_protocol(int f_out,int f_in) |
f9df736a WD |
25 | if (read_batch) |
26 | check_batch_flags(); | |
27 | ||
28 | +#ifndef SUPPORT_PREALLOCATION | |
29 | + if (preallocate_files && !am_sender) { | |
30 | + rprintf(FERROR, | |
31 | + "preallocation is not supported on this %s\n", | |
32 | + am_server ? "server" : "client"); | |
33 | + exit_cleanup(RERR_SYNTAX); | |
34 | + } | |
35 | +#endif | |
36 | + | |
37 | if (protocol_version < 30) { | |
38 | if (append_mode == 1) | |
39 | append_mode = 2; | |
cc3e685d | 40 | diff --git a/configure.in b/configure.in |
fc557362 | 41 | index bc7d4a7..61f9b05 100644 |
cc3e685d WD |
42 | --- a/configure.in |
43 | +++ b/configure.in | |
fc557362 | 44 | @@ -553,13 +553,40 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ |
5e3c6c93 | 45 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ |
80c89075 | 46 | strerror putenv iconv_open locale_charset nl_langinfo getxattr \ |
fc557362 WD |
47 | extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ |
48 | - initgroups) | |
49 | + initgroups fallocate posix_fallocate) | |
5e3c6c93 | 50 | |
4c15e800 WD |
51 | dnl cygwin iconv.h defines iconv_open as libiconv_open |
52 | if test x"$ac_cv_func_iconv_open" != x"yes"; then | |
f9df736a WD |
53 | AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) |
54 | fi | |
55 | ||
56 | +dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): | |
57 | + | |
58 | +AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ | |
59 | +AC_TRY_COMPILE([#include <sys/syscall.h> | |
60 | +#include <sys/types.h>], | |
61 | +[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);], | |
62 | +rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) | |
63 | +if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then | |
64 | + AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) | |
65 | +fi | |
66 | + | |
67 | +if test x"$ac_cv_func_posix_fallocate" = x"yes"; then | |
68 | + AC_MSG_CHECKING([whether posix_fallocate is efficient]) | |
69 | + case $host_os in | |
70 | + *cygwin*) | |
71 | + AC_MSG_RESULT(yes) | |
72 | + AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, | |
73 | + [Define if posix_fallocate is efficient (Cygwin)]) | |
74 | + ;; | |
75 | + *) | |
76 | + AC_MSG_RESULT(no) | |
77 | + ;; | |
78 | + esac | |
79 | +fi | |
80 | + | |
81 | +dnl End of preallocation stuff | |
82 | + | |
83 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) | |
84 | if test $ac_cv_func_getpgrp = yes; then | |
85 | AC_FUNC_GETPGRP | |
cc3e685d | 86 | diff --git a/options.c b/options.c |
fc557362 | 87 | index e7c6c61..6f848f1 100644 |
cc3e685d WD |
88 | --- a/options.c |
89 | +++ b/options.c | |
c0c7984e | 90 | @@ -73,6 +73,7 @@ int remove_source_files = 0; |
5e3c6c93 WD |
91 | int one_file_system = 0; |
92 | int protocol_version = PROTOCOL_VERSION; | |
93 | int sparse_files = 0; | |
94 | +int preallocate_files = 0; | |
95 | int do_compression = 0; | |
96 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
58b399b9 | 97 | int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ |
fc557362 | 98 | @@ -566,6 +567,7 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 | 99 | char const *links = "no "; |
58b399b9 | 100 | char const *iconv = "no "; |
5e3c6c93 WD |
101 | char const *ipv6 = "no "; |
102 | + char const *preallocation = "no "; | |
103 | STRUCT_STAT *dumstat; | |
104 | ||
ac2da598 | 105 | #if SUBPROTOCOL_VERSION != 0 |
fc557362 | 106 | @@ -599,6 +601,9 @@ static void print_rsync_version(enum logcode f) |
85096e5e WD |
107 | #if defined HAVE_LUTIMES && defined HAVE_UTIMES |
108 | symtimes = ""; | |
5e3c6c93 | 109 | #endif |
5e3c6c93 WD |
110 | +#ifdef SUPPORT_PREALLOCATION |
111 | + preallocation = ""; | |
112 | +#endif | |
ac2da598 WD |
113 | |
114 | rprintf(f, "%s version %s protocol version %d%s\n", | |
115 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
fc557362 | 116 | @@ -612,8 +617,8 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 WD |
117 | (int)(sizeof (int64) * 8)); |
118 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
119 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
85096e5e WD |
120 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", |
121 | - have_inplace, acls, xattrs, iconv, symtimes); | |
122 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %spreallocation\n", | |
123 | + have_inplace, acls, xattrs, iconv, symtimes, preallocation); | |
5e3c6c93 WD |
124 | |
125 | #ifdef MAINTAINER_MODE | |
126 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
fc557362 | 127 | @@ -703,6 +708,9 @@ void usage(enum logcode F) |
58b399b9 WD |
128 | rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); |
129 | #endif | |
5e3c6c93 WD |
130 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
131 | +#ifdef SUPPORT_PREALLOCATION | |
f9df736a | 132 | + rprintf(F," --preallocate allocate dest files before writing them\n"); |
5e3c6c93 | 133 | +#endif |
e2b0842a | 134 | rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); |
f2863bc0 | 135 | rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); |
5e3c6c93 | 136 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); |
fc557362 | 137 | @@ -899,6 +907,7 @@ static struct poptOption long_options[] = { |
c0c7984e WD |
138 | {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, |
139 | {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
140 | {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
5e3c6c93 | 141 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, |
c0c7984e WD |
142 | {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, |
143 | {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, | |
790ba11a | 144 | {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, |
fc557362 | 145 | @@ -2609,6 +2618,9 @@ void server_options(char **args, int *argc_p) |
5e3c6c93 WD |
146 | else if (remove_source_files) |
147 | args[ac++] = "--remove-sent-files"; | |
148 | ||
149 | + if (preallocate_files && am_sender) | |
150 | + args[ac++] = "--preallocate"; | |
151 | + | |
ae306a29 WD |
152 | if (ac > MAX_SERVER_ARGS) { /* Not possible... */ |
153 | rprintf(FERROR, "argc overflow in server_options().\n"); | |
154 | exit_cleanup(RERR_MALLOC); | |
cc3e685d | 155 | diff --git a/receiver.c b/receiver.c |
fc557362 | 156 | index 4325e30..739a0ba 100644 |
cc3e685d WD |
157 | --- a/receiver.c |
158 | +++ b/receiver.c | |
fc557362 | 159 | @@ -44,6 +44,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
160 | extern int remove_source_files; |
161 | extern int append_mode; | |
162 | extern int sparse_files; | |
163 | +extern int preallocate_files; | |
164 | extern int keep_partial; | |
fc557362 | 165 | extern int checksum_len; |
5e3c6c93 | 166 | extern int checksum_seed; |
963ca808 | 167 | @@ -175,6 +176,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
f9df736a | 168 | char *data; |
5e3c6c93 WD |
169 | int32 i; |
170 | char *map = NULL; | |
5e3c6c93 | 171 | +#ifdef SUPPORT_PREALLOCATION |
cc3e685d | 172 | + OFF_T preallocated_len = 0; |
5e3c6c93 WD |
173 | + |
174 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
175 | + /* Preallocate enough space for file's eventual length if | |
176 | + * possible; seems to reduce fragmentation on Windows. */ | |
f9df736a | 177 | + if (do_fallocate(fd, 0, total_size) == 0) |
5e3c6c93 WD |
178 | + preallocated_len = total_size; |
179 | + else | |
f9df736a | 180 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); |
5e3c6c93 WD |
181 | + } |
182 | +#endif | |
f9df736a | 183 | |
5e3c6c93 WD |
184 | read_sum_head(f_in, &sum); |
185 | ||
fc557362 | 186 | @@ -285,8 +298,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
5e3c6c93 WD |
187 | goto report_write_error; |
188 | ||
189 | #ifdef HAVE_FTRUNCATE | |
abd3adb8 | 190 | - if (inplace && fd != -1 |
fc557362 | 191 | - && ftruncate(fd, offset) < 0) { |
5e3c6c93 WD |
192 | + /* inplace: New data could be shorter than old data. |
193 | + * preallocate_files: total_size could have been an overestimate. | |
194 | + * Cut off any extra preallocated zeros from dest file. */ | |
fc557362 | 195 | + if ((inplace |
5e3c6c93 | 196 | +#ifdef SUPPORT_PREALLOCATION |
fc557362 | 197 | + || preallocated_len > offset |
5e3c6c93 | 198 | +#endif |
fc557362 | 199 | + ) && fd != -1 && ftruncate(fd, offset) < 0) { |
abd3adb8 WD |
200 | rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", |
201 | full_fname(fname)); | |
fc557362 | 202 | } |
cc3e685d | 203 | diff --git a/rsync.h b/rsync.h |
fc557362 | 204 | index be7cf8a..0ad3075 100644 |
cc3e685d WD |
205 | --- a/rsync.h |
206 | +++ b/rsync.h | |
fc557362 | 207 | @@ -634,6 +634,13 @@ struct ht_int64_node { |
ffc18846 | 208 | #define ACLS_NEED_MASK 1 |
5e3c6c93 WD |
209 | #endif |
210 | ||
f9df736a WD |
211 | +#if defined HAVE_FTRUNCATE \ |
212 | + && (defined HAVE_FALLOCATE \ | |
213 | + || defined HAVE_SYS_FALLOCATE \ | |
214 | + || defined HAVE_EFFICIENT_POSIX_FALLOCATE) | |
5e3c6c93 WD |
215 | +#define SUPPORT_PREALLOCATION 1 |
216 | +#endif | |
217 | + | |
612d3765 | 218 | union file_extras { |
c4bd76ea WD |
219 | int32 num; |
220 | uint32 unum; | |
cc3e685d | 221 | diff --git a/rsync.yo b/rsync.yo |
fc557362 | 222 | index 941f7a5..36a2077 100644 |
cc3e685d WD |
223 | --- a/rsync.yo |
224 | +++ b/rsync.yo | |
fc557362 | 225 | @@ -359,6 +359,7 @@ to the detailed description below for a complete description. verb( |
612d3765 | 226 | --super receiver attempts super-user activities |
58b399b9 | 227 | --fake-super store/recover privileged attrs using xattrs |
612d3765 | 228 | -S, --sparse handle sparse files efficiently |
f9df736a | 229 | + --preallocate allocate dest files before writing |
e2b0842a | 230 | -n, --dry-run perform a trial run with no changes made |
f2863bc0 | 231 | -W, --whole-file copy files whole (w/o delta-xfer algorithm) |
612d3765 | 232 | -x, --one-file-system don't cross filesystem boundaries |
fc557362 | 233 | @@ -1120,6 +1121,18 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" |
612d3765 WD |
234 | filesystem. It doesn't seem to handle seeks over null regions |
235 | correctly and ends up corrupting the files. | |
236 | ||
237 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
f9df736a WD |
238 | +file to its eventual size before writing data to the file. Rsync will only use |
239 | +the real filesystem-level preallocation support provided by bf(fallocate)(2) or | |
240 | +Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes | |
241 | +a zero byte into each block. If the receiver is remote, this nonstandard | |
242 | +option only works if the receiver also has the preallocation patch. | |
612d3765 WD |
243 | + |
244 | +Without this option on MS Windows, very large destination files tend to be | |
245 | +broken into thousands of fragments; advising Windows ahead of time of the | |
246 | +eventual file size using this option usually reduces the number of | |
f9df736a | 247 | +fragments to one. The usefulness of this option on Linux is yet to be tested. |
612d3765 | 248 | + |
e2b0842a WD |
249 | dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't |
250 | make any changes (and produces mostly the same output as a real run). It | |
251 | is most commonly used in combination with the bf(-v, --verbose) and/or | |
f9df736a | 252 | diff --git a/syscall.c b/syscall.c |
fc557362 | 253 | index cfabc3e..81fb957 100644 |
f9df736a WD |
254 | --- a/syscall.c |
255 | +++ b/syscall.c | |
256 | @@ -29,6 +29,10 @@ | |
257 | #include <sys/attr.h> | |
258 | #endif | |
259 | ||
260 | +#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE | |
261 | +#include <sys/syscall.h> | |
262 | +#endif | |
263 | + | |
264 | extern int dry_run; | |
265 | extern int am_root; | |
266 | extern int read_only; | |
c0c7984e WD |
267 | @@ -282,3 +286,21 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence) |
268 | return lseek(fd, offset, whence); | |
f9df736a WD |
269 | #endif |
270 | } | |
271 | + | |
272 | +#ifdef SUPPORT_PREALLOCATION | |
273 | +int do_fallocate(int fd, OFF_T offset, OFF_T length) | |
274 | +{ | |
275 | + RETURN_ERROR_IF(dry_run, 0); | |
276 | + RETURN_ERROR_IF_RO_OR_LO; | |
277 | + /* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */ | |
278 | +#if defined HAVE_FALLOCATE | |
279 | + return fallocate(fd, 0, offset, length); | |
280 | +#elif defined HAVE_SYS_FALLOCATE | |
281 | + return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length); | |
282 | +#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE | |
283 | + return posix_fallocate(fd, offset, length); | |
284 | +#else | |
285 | +#error coding error in SUPPORT_PREALLOCATION | |
286 | +#endif | |
287 | +} | |
288 | +#endif | |
cc3e685d | 289 | diff --git a/t_stub.c b/t_stub.c |
fc557362 | 290 | index 02cfa69..52a7f02 100644 |
cc3e685d WD |
291 | --- a/t_stub.c |
292 | +++ b/t_stub.c | |
ffc18846 | 293 | @@ -22,6 +22,7 @@ |
5e3c6c93 WD |
294 | #include "rsync.h" |
295 | ||
296 | int modify_window = 0; | |
297 | +int preallocate_files = 0; | |
298 | int module_id = -1; | |
299 | int relative_paths = 0; | |
fc557362 | 300 | int module_dirlen = 0; |
cc3e685d | 301 | diff --git a/util.c b/util.c |
fc557362 | 302 | index 0cafed6..20dd0d3 100644 |
cc3e685d WD |
303 | --- a/util.c |
304 | +++ b/util.c | |
fc557362 WD |
305 | @@ -26,6 +26,7 @@ |
306 | #include "inums.h" | |
5e3c6c93 | 307 | |
5e3c6c93 WD |
308 | extern int dry_run; |
309 | +extern int preallocate_files; | |
310 | extern int module_id; | |
311 | extern int modify_window; | |
312 | extern int relative_paths; | |
91270139 | 313 | @@ -276,6 +277,10 @@ int copy_file(const char *source, const char *dest, int ofd, |
e2b0842a | 314 | int ifd; |
5e3c6c93 WD |
315 | char buf[1024 * 8]; |
316 | int len; /* Number of bytes read into `buf'. */ | |
317 | +#ifdef SUPPORT_PREALLOCATION | |
cc3e685d WD |
318 | + OFF_T preallocated_len = 0; |
319 | + OFF_T offset = 0; | |
5e3c6c93 WD |
320 | +#endif |
321 | ||
c8a8b4a7 | 322 | if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { |
91270139 WD |
323 | int save_errno = errno; |
324 | @@ -309,7 +314,27 @@ int copy_file(const char *source, const char *dest, int ofd, | |
e2b0842a | 325 | } |
5e3c6c93 WD |
326 | } |
327 | ||
328 | +#ifdef SUPPORT_PREALLOCATION | |
329 | + if (preallocate_files) { | |
330 | + /* Preallocate enough space for file's eventual length if | |
331 | + * possible; seems to reduce fragmentation on Windows. */ | |
332 | + STRUCT_STAT srcst; | |
333 | + if (do_fstat(ifd, &srcst) == 0) { | |
334 | + if (srcst.st_size > 0) { | |
f9df736a | 335 | + if (do_fallocate(ofd, 0, srcst.st_size) == 0) |
5e3c6c93 WD |
336 | + preallocated_len = srcst.st_size; |
337 | + else | |
f9df736a | 338 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); |
5e3c6c93 WD |
339 | + } |
340 | + } else | |
f9df736a | 341 | + rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); |
5e3c6c93 WD |
342 | + } |
343 | +#endif | |
344 | + | |
345 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
346 | +#ifdef SUPPORT_PREALLOCATION | |
347 | + offset += len; | |
348 | +#endif | |
349 | if (full_write(ofd, buf, len) < 0) { | |
91270139 | 350 | int save_errno = errno; |
cc3e685d | 351 | rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest)); |
91270139 | 352 | @@ -334,6 +359,16 @@ int copy_file(const char *source, const char *dest, int ofd, |
5e3c6c93 WD |
353 | full_fname(source)); |
354 | } | |
355 | ||
356 | +#ifdef SUPPORT_PREALLOCATION | |
357 | + /* Source file might have shrunk since we fstatted it. | |
358 | + * Cut off any extra preallocated zeros from dest file. */ | |
359 | + if (preallocated_len > offset) | |
360 | + if (ftruncate(ofd, offset) < 0) | |
361 | + /* If we fail to truncate, the dest file may be wrong, so we | |
362 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 363 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); |
5e3c6c93 WD |
364 | +#endif |
365 | + | |
366 | if (close(ofd) < 0) { | |
91270139 | 367 | int save_errno = errno; |
cc3e685d | 368 | rsyserr(FERROR_XFER, errno, "close failed on %s", |