Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
72e5645e | 12 | based-on: 3b8f8192227b14e708bf535072485e50f4362270 |
f9df736a WD |
13 | diff --git a/compat.c b/compat.c |
14 | --- a/compat.c | |
15 | +++ b/compat.c | |
72e5645e | 16 | @@ -34,6 +34,7 @@ extern int inplace; |
f9df736a WD |
17 | extern int recurse; |
18 | extern int use_qsort; | |
19 | extern int allow_inc_recurse; | |
20 | +extern int preallocate_files; | |
21 | extern int append_mode; | |
22 | extern int fuzzy_basis; | |
23 | extern int read_batch; | |
72e5645e | 24 | @@ -189,6 +190,15 @@ void setup_protocol(int f_out,int f_in) |
f9df736a WD |
25 | if (read_batch) |
26 | check_batch_flags(); | |
27 | ||
28 | +#ifndef SUPPORT_PREALLOCATION | |
29 | + if (preallocate_files && !am_sender) { | |
30 | + rprintf(FERROR, | |
31 | + "preallocation is not supported on this %s\n", | |
32 | + am_server ? "server" : "client"); | |
33 | + exit_cleanup(RERR_SYNTAX); | |
34 | + } | |
35 | +#endif | |
36 | + | |
37 | if (protocol_version < 30) { | |
38 | if (append_mode == 1) | |
39 | append_mode = 2; | |
cc3e685d WD |
40 | diff --git a/configure.in b/configure.in |
41 | --- a/configure.in | |
42 | +++ b/configure.in | |
72e5645e | 43 | @@ -574,13 +574,40 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ |
5e3c6c93 | 44 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ |
80c89075 | 45 | strerror putenv iconv_open locale_charset nl_langinfo getxattr \ |
fc557362 | 46 | extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ |
72e5645e WD |
47 | - initgroups utimensat) |
48 | + initgroups utimensat fallocate posix_fallocate) | |
5e3c6c93 | 49 | |
4c15e800 WD |
50 | dnl cygwin iconv.h defines iconv_open as libiconv_open |
51 | if test x"$ac_cv_func_iconv_open" != x"yes"; then | |
f9df736a WD |
52 | AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) |
53 | fi | |
54 | ||
55 | +dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): | |
56 | + | |
57 | +AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ | |
58 | +AC_TRY_COMPILE([#include <sys/syscall.h> | |
59 | +#include <sys/types.h>], | |
60 | +[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);], | |
61 | +rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) | |
62 | +if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then | |
63 | + AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) | |
64 | +fi | |
65 | + | |
66 | +if test x"$ac_cv_func_posix_fallocate" = x"yes"; then | |
67 | + AC_MSG_CHECKING([whether posix_fallocate is efficient]) | |
68 | + case $host_os in | |
69 | + *cygwin*) | |
70 | + AC_MSG_RESULT(yes) | |
71 | + AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, | |
72 | + [Define if posix_fallocate is efficient (Cygwin)]) | |
73 | + ;; | |
74 | + *) | |
75 | + AC_MSG_RESULT(no) | |
76 | + ;; | |
77 | + esac | |
78 | +fi | |
79 | + | |
80 | +dnl End of preallocation stuff | |
81 | + | |
82 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) | |
83 | if test $ac_cv_func_getpgrp = yes; then | |
84 | AC_FUNC_GETPGRP | |
cc3e685d WD |
85 | diff --git a/options.c b/options.c |
86 | --- a/options.c | |
87 | +++ b/options.c | |
c0c7984e | 88 | @@ -73,6 +73,7 @@ int remove_source_files = 0; |
5e3c6c93 WD |
89 | int one_file_system = 0; |
90 | int protocol_version = PROTOCOL_VERSION; | |
91 | int sparse_files = 0; | |
92 | +int preallocate_files = 0; | |
93 | int do_compression = 0; | |
94 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
58b399b9 | 95 | int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ |
72e5645e | 96 | @@ -567,6 +568,7 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 | 97 | char const *links = "no "; |
58b399b9 | 98 | char const *iconv = "no "; |
5e3c6c93 WD |
99 | char const *ipv6 = "no "; |
100 | + char const *preallocation = "no "; | |
101 | STRUCT_STAT *dumstat; | |
102 | ||
ac2da598 | 103 | #if SUBPROTOCOL_VERSION != 0 |
72e5645e WD |
104 | @@ -600,6 +602,9 @@ static void print_rsync_version(enum logcode f) |
105 | #ifdef CAN_SET_SYMLINK_TIMES | |
85096e5e | 106 | symtimes = ""; |
5e3c6c93 | 107 | #endif |
5e3c6c93 WD |
108 | +#ifdef SUPPORT_PREALLOCATION |
109 | + preallocation = ""; | |
110 | +#endif | |
ac2da598 WD |
111 | |
112 | rprintf(f, "%s version %s protocol version %d%s\n", | |
113 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
72e5645e | 114 | @@ -613,8 +618,8 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 WD |
115 | (int)(sizeof (int64) * 8)); |
116 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
117 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
85096e5e WD |
118 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", |
119 | - have_inplace, acls, xattrs, iconv, symtimes); | |
120 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %spreallocation\n", | |
121 | + have_inplace, acls, xattrs, iconv, symtimes, preallocation); | |
5e3c6c93 WD |
122 | |
123 | #ifdef MAINTAINER_MODE | |
124 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
72e5645e | 125 | @@ -704,6 +709,9 @@ void usage(enum logcode F) |
58b399b9 WD |
126 | rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); |
127 | #endif | |
5e3c6c93 WD |
128 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
129 | +#ifdef SUPPORT_PREALLOCATION | |
f9df736a | 130 | + rprintf(F," --preallocate allocate dest files before writing them\n"); |
5e3c6c93 | 131 | +#endif |
e2b0842a | 132 | rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); |
f2863bc0 | 133 | rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); |
5e3c6c93 | 134 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); |
72e5645e | 135 | @@ -900,6 +908,7 @@ static struct poptOption long_options[] = { |
c0c7984e WD |
136 | {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, |
137 | {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
138 | {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, | |
5e3c6c93 | 139 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, |
c0c7984e WD |
140 | {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, |
141 | {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, | |
790ba11a | 142 | {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, |
72e5645e | 143 | @@ -2627,6 +2636,9 @@ void server_options(char **args, int *argc_p) |
5e3c6c93 WD |
144 | else if (remove_source_files) |
145 | args[ac++] = "--remove-sent-files"; | |
146 | ||
147 | + if (preallocate_files && am_sender) | |
148 | + args[ac++] = "--preallocate"; | |
149 | + | |
ae306a29 WD |
150 | if (ac > MAX_SERVER_ARGS) { /* Not possible... */ |
151 | rprintf(FERROR, "argc overflow in server_options().\n"); | |
152 | exit_cleanup(RERR_MALLOC); | |
cc3e685d WD |
153 | diff --git a/receiver.c b/receiver.c |
154 | --- a/receiver.c | |
155 | +++ b/receiver.c | |
fc557362 | 156 | @@ -44,6 +44,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
157 | extern int remove_source_files; |
158 | extern int append_mode; | |
159 | extern int sparse_files; | |
160 | +extern int preallocate_files; | |
161 | extern int keep_partial; | |
fc557362 | 162 | extern int checksum_len; |
5e3c6c93 | 163 | extern int checksum_seed; |
72e5645e | 164 | @@ -207,6 +208,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
f9df736a | 165 | char *data; |
5e3c6c93 WD |
166 | int32 i; |
167 | char *map = NULL; | |
5e3c6c93 | 168 | +#ifdef SUPPORT_PREALLOCATION |
cc3e685d | 169 | + OFF_T preallocated_len = 0; |
5e3c6c93 WD |
170 | + |
171 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
172 | + /* Preallocate enough space for file's eventual length if | |
173 | + * possible; seems to reduce fragmentation on Windows. */ | |
f9df736a | 174 | + if (do_fallocate(fd, 0, total_size) == 0) |
5e3c6c93 WD |
175 | + preallocated_len = total_size; |
176 | + else | |
f9df736a | 177 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); |
5e3c6c93 WD |
178 | + } |
179 | +#endif | |
f9df736a | 180 | |
5e3c6c93 WD |
181 | read_sum_head(f_in, &sum); |
182 | ||
72e5645e | 183 | @@ -317,8 +330,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
5e3c6c93 WD |
184 | goto report_write_error; |
185 | ||
186 | #ifdef HAVE_FTRUNCATE | |
abd3adb8 | 187 | - if (inplace && fd != -1 |
fc557362 | 188 | - && ftruncate(fd, offset) < 0) { |
5e3c6c93 WD |
189 | + /* inplace: New data could be shorter than old data. |
190 | + * preallocate_files: total_size could have been an overestimate. | |
191 | + * Cut off any extra preallocated zeros from dest file. */ | |
fc557362 | 192 | + if ((inplace |
5e3c6c93 | 193 | +#ifdef SUPPORT_PREALLOCATION |
fc557362 | 194 | + || preallocated_len > offset |
5e3c6c93 | 195 | +#endif |
fc557362 | 196 | + ) && fd != -1 && ftruncate(fd, offset) < 0) { |
abd3adb8 WD |
197 | rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", |
198 | full_fname(fname)); | |
fc557362 | 199 | } |
cc3e685d WD |
200 | diff --git a/rsync.h b/rsync.h |
201 | --- a/rsync.h | |
202 | +++ b/rsync.h | |
72e5645e | 203 | @@ -642,6 +642,13 @@ struct ht_int64_node { |
ffc18846 | 204 | #define ACLS_NEED_MASK 1 |
5e3c6c93 WD |
205 | #endif |
206 | ||
f9df736a WD |
207 | +#if defined HAVE_FTRUNCATE \ |
208 | + && (defined HAVE_FALLOCATE \ | |
209 | + || defined HAVE_SYS_FALLOCATE \ | |
210 | + || defined HAVE_EFFICIENT_POSIX_FALLOCATE) | |
5e3c6c93 WD |
211 | +#define SUPPORT_PREALLOCATION 1 |
212 | +#endif | |
213 | + | |
612d3765 | 214 | union file_extras { |
c4bd76ea WD |
215 | int32 num; |
216 | uint32 unum; | |
cc3e685d WD |
217 | diff --git a/rsync.yo b/rsync.yo |
218 | --- a/rsync.yo | |
219 | +++ b/rsync.yo | |
fc557362 | 220 | @@ -359,6 +359,7 @@ to the detailed description below for a complete description. verb( |
612d3765 | 221 | --super receiver attempts super-user activities |
58b399b9 | 222 | --fake-super store/recover privileged attrs using xattrs |
612d3765 | 223 | -S, --sparse handle sparse files efficiently |
f9df736a | 224 | + --preallocate allocate dest files before writing |
e2b0842a | 225 | -n, --dry-run perform a trial run with no changes made |
f2863bc0 | 226 | -W, --whole-file copy files whole (w/o delta-xfer algorithm) |
612d3765 | 227 | -x, --one-file-system don't cross filesystem boundaries |
7170ca8d | 228 | @@ -1127,6 +1128,18 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" |
72e5645e WD |
229 | filesystem. It seems to have problems seeking over null regions, |
230 | and ends up corrupting the files. | |
612d3765 WD |
231 | |
232 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
f9df736a WD |
233 | +file to its eventual size before writing data to the file. Rsync will only use |
234 | +the real filesystem-level preallocation support provided by bf(fallocate)(2) or | |
235 | +Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes | |
236 | +a zero byte into each block. If the receiver is remote, this nonstandard | |
237 | +option only works if the receiver also has the preallocation patch. | |
612d3765 WD |
238 | + |
239 | +Without this option on MS Windows, very large destination files tend to be | |
240 | +broken into thousands of fragments; advising Windows ahead of time of the | |
241 | +eventual file size using this option usually reduces the number of | |
f9df736a | 242 | +fragments to one. The usefulness of this option on Linux is yet to be tested. |
612d3765 | 243 | + |
e2b0842a WD |
244 | dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't |
245 | make any changes (and produces mostly the same output as a real run). It | |
246 | is most commonly used in combination with the bf(-v, --verbose) and/or | |
f9df736a WD |
247 | diff --git a/syscall.c b/syscall.c |
248 | --- a/syscall.c | |
249 | +++ b/syscall.c | |
250 | @@ -29,6 +29,10 @@ | |
251 | #include <sys/attr.h> | |
252 | #endif | |
253 | ||
254 | +#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE | |
255 | +#include <sys/syscall.h> | |
256 | +#endif | |
257 | + | |
258 | extern int dry_run; | |
259 | extern int am_root; | |
72e5645e WD |
260 | extern int am_sender; |
261 | @@ -325,3 +329,21 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence) | |
c0c7984e | 262 | return lseek(fd, offset, whence); |
f9df736a WD |
263 | #endif |
264 | } | |
265 | + | |
266 | +#ifdef SUPPORT_PREALLOCATION | |
267 | +int do_fallocate(int fd, OFF_T offset, OFF_T length) | |
268 | +{ | |
269 | + RETURN_ERROR_IF(dry_run, 0); | |
270 | + RETURN_ERROR_IF_RO_OR_LO; | |
271 | + /* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */ | |
272 | +#if defined HAVE_FALLOCATE | |
273 | + return fallocate(fd, 0, offset, length); | |
274 | +#elif defined HAVE_SYS_FALLOCATE | |
275 | + return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length); | |
276 | +#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE | |
277 | + return posix_fallocate(fd, offset, length); | |
278 | +#else | |
279 | +#error coding error in SUPPORT_PREALLOCATION | |
280 | +#endif | |
281 | +} | |
282 | +#endif | |
cc3e685d WD |
283 | diff --git a/t_stub.c b/t_stub.c |
284 | --- a/t_stub.c | |
285 | +++ b/t_stub.c | |
ffc18846 | 286 | @@ -22,6 +22,7 @@ |
5e3c6c93 WD |
287 | #include "rsync.h" |
288 | ||
289 | int modify_window = 0; | |
290 | +int preallocate_files = 0; | |
291 | int module_id = -1; | |
292 | int relative_paths = 0; | |
fc557362 | 293 | int module_dirlen = 0; |
cc3e685d WD |
294 | diff --git a/util.c b/util.c |
295 | --- a/util.c | |
296 | +++ b/util.c | |
fc557362 WD |
297 | @@ -26,6 +26,7 @@ |
298 | #include "inums.h" | |
5e3c6c93 | 299 | |
5e3c6c93 WD |
300 | extern int dry_run; |
301 | +extern int preallocate_files; | |
302 | extern int module_id; | |
303 | extern int modify_window; | |
304 | extern int relative_paths; | |
72e5645e | 305 | @@ -332,6 +333,10 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) |
e2b0842a | 306 | int ifd; |
5e3c6c93 WD |
307 | char buf[1024 * 8]; |
308 | int len; /* Number of bytes read into `buf'. */ | |
309 | +#ifdef SUPPORT_PREALLOCATION | |
cc3e685d WD |
310 | + OFF_T preallocated_len = 0; |
311 | + OFF_T offset = 0; | |
5e3c6c93 WD |
312 | +#endif |
313 | ||
c8a8b4a7 | 314 | if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { |
91270139 | 315 | int save_errno = errno; |
72e5645e | 316 | @@ -357,7 +362,27 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) |
e2b0842a | 317 | } |
5e3c6c93 WD |
318 | } |
319 | ||
320 | +#ifdef SUPPORT_PREALLOCATION | |
321 | + if (preallocate_files) { | |
322 | + /* Preallocate enough space for file's eventual length if | |
323 | + * possible; seems to reduce fragmentation on Windows. */ | |
324 | + STRUCT_STAT srcst; | |
325 | + if (do_fstat(ifd, &srcst) == 0) { | |
326 | + if (srcst.st_size > 0) { | |
f9df736a | 327 | + if (do_fallocate(ofd, 0, srcst.st_size) == 0) |
5e3c6c93 WD |
328 | + preallocated_len = srcst.st_size; |
329 | + else | |
f9df736a | 330 | + rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); |
5e3c6c93 WD |
331 | + } |
332 | + } else | |
f9df736a | 333 | + rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); |
5e3c6c93 WD |
334 | + } |
335 | +#endif | |
336 | + | |
337 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
338 | +#ifdef SUPPORT_PREALLOCATION | |
339 | + offset += len; | |
340 | +#endif | |
341 | if (full_write(ofd, buf, len) < 0) { | |
91270139 | 342 | int save_errno = errno; |
cc3e685d | 343 | rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest)); |
72e5645e | 344 | @@ -382,6 +407,16 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) |
5e3c6c93 WD |
345 | full_fname(source)); |
346 | } | |
347 | ||
348 | +#ifdef SUPPORT_PREALLOCATION | |
349 | + /* Source file might have shrunk since we fstatted it. | |
350 | + * Cut off any extra preallocated zeros from dest file. */ | |
351 | + if (preallocated_len > offset) | |
352 | + if (ftruncate(ofd, offset) < 0) | |
353 | + /* If we fail to truncate, the dest file may be wrong, so we | |
354 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 355 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); |
5e3c6c93 WD |
356 | +#endif |
357 | + | |
358 | if (close(ofd) < 0) { | |
91270139 | 359 | int save_errno = errno; |
cc3e685d | 360 | rsyserr(FERROR_XFER, errno, "close failed on %s", |