Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
cc3e685d WD |
12 | diff --git a/configure.in b/configure.in |
13 | --- a/configure.in | |
14 | +++ b/configure.in | |
15 | @@ -552,7 +552,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ | |
5e3c6c93 WD |
16 | strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ |
17 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ | |
80c89075 | 18 | strerror putenv iconv_open locale_charset nl_langinfo getxattr \ |
7c4c2959 WD |
19 | - extattr_get_link sigaction sigprocmask setattrlist) |
20 | + extattr_get_link sigaction sigprocmask setattrlist posix_fallocate) | |
5e3c6c93 | 21 | |
4c15e800 WD |
22 | dnl cygwin iconv.h defines iconv_open as libiconv_open |
23 | if test x"$ac_cv_func_iconv_open" != x"yes"; then | |
cc3e685d WD |
24 | diff --git a/options.c b/options.c |
25 | --- a/options.c | |
26 | +++ b/options.c | |
f2863bc0 | 27 | @@ -72,6 +72,7 @@ int remove_source_files = 0; |
5e3c6c93 WD |
28 | int one_file_system = 0; |
29 | int protocol_version = PROTOCOL_VERSION; | |
30 | int sparse_files = 0; | |
31 | +int preallocate_files = 0; | |
32 | int do_compression = 0; | |
33 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
58b399b9 | 34 | int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ |
f2863bc0 | 35 | @@ -223,6 +224,7 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 | 36 | char const *links = "no "; |
58b399b9 | 37 | char const *iconv = "no "; |
5e3c6c93 WD |
38 | char const *ipv6 = "no "; |
39 | + char const *preallocation = "no "; | |
40 | STRUCT_STAT *dumstat; | |
41 | ||
ac2da598 | 42 | #if SUBPROTOCOL_VERSION != 0 |
f2863bc0 | 43 | @@ -252,6 +254,9 @@ static void print_rsync_version(enum logcode f) |
58b399b9 WD |
44 | #ifdef ICONV_OPTION |
45 | iconv = ""; | |
5e3c6c93 | 46 | #endif |
5e3c6c93 WD |
47 | +#ifdef SUPPORT_PREALLOCATION |
48 | + preallocation = ""; | |
49 | +#endif | |
ac2da598 WD |
50 | |
51 | rprintf(f, "%s version %s protocol version %d%s\n", | |
52 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
f2863bc0 | 53 | @@ -265,8 +270,8 @@ static void print_rsync_version(enum logcode f) |
5e3c6c93 WD |
54 | (int)(sizeof (int64) * 8)); |
55 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
56 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
58b399b9 WD |
57 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv\n", |
58 | - have_inplace, acls, xattrs, iconv); | |
59 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %spreallocation\n", | |
60 | + have_inplace, acls, xattrs, iconv, preallocation); | |
5e3c6c93 WD |
61 | |
62 | #ifdef MAINTAINER_MODE | |
63 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
f2863bc0 | 64 | @@ -353,6 +358,9 @@ void usage(enum logcode F) |
58b399b9 WD |
65 | rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); |
66 | #endif | |
5e3c6c93 WD |
67 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
68 | +#ifdef SUPPORT_PREALLOCATION | |
69 | + rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); | |
70 | +#endif | |
e2b0842a | 71 | rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); |
f2863bc0 | 72 | rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); |
5e3c6c93 | 73 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); |
f2863bc0 | 74 | @@ -531,6 +539,7 @@ static struct poptOption long_options[] = { |
5e3c6c93 WD |
75 | {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, |
76 | {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, | |
77 | {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, | |
78 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, | |
79 | {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, | |
790ba11a WD |
80 | {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, |
81 | {"append-verify", 0, POPT_ARG_VAL, &append_mode, 2, 0, 0 }, | |
f2863bc0 | 82 | @@ -1289,6 +1298,15 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) |
5e3c6c93 WD |
83 | } |
84 | #endif | |
85 | ||
86 | +#ifndef SUPPORT_PREALLOCATION | |
87 | + if (preallocate_files && !am_sender) { | |
88 | + snprintf(err_buf, sizeof err_buf, | |
89 | + "preallocation is not supported on this %s\n", | |
90 | + am_server ? "server" : "client"); | |
91 | + return 0; | |
92 | + } | |
93 | +#endif | |
94 | + | |
95 | if (write_batch && read_batch) { | |
96 | snprintf(err_buf, sizeof err_buf, | |
97 | "--write-batch and --read-batch can not be used together\n"); | |
a5e6228a | 98 | @@ -2000,6 +2018,9 @@ void server_options(char **args, int *argc_p) |
5e3c6c93 WD |
99 | else if (remove_source_files) |
100 | args[ac++] = "--remove-sent-files"; | |
101 | ||
102 | + if (preallocate_files && am_sender) | |
103 | + args[ac++] = "--preallocate"; | |
104 | + | |
790ba11a | 105 | *argc_p = ac; |
5e3c6c93 WD |
106 | return; |
107 | ||
cc3e685d WD |
108 | diff --git a/receiver.c b/receiver.c |
109 | --- a/receiver.c | |
110 | +++ b/receiver.c | |
5795bf59 | 111 | @@ -45,6 +45,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
112 | extern int remove_source_files; |
113 | extern int append_mode; | |
114 | extern int sparse_files; | |
115 | +extern int preallocate_files; | |
116 | extern int keep_partial; | |
117 | extern int checksum_seed; | |
118 | extern int inplace; | |
cc3e685d | 119 | @@ -175,6 +176,19 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
5e3c6c93 WD |
120 | int32 i; |
121 | char *map = NULL; | |
122 | ||
123 | +#ifdef SUPPORT_PREALLOCATION | |
cc3e685d | 124 | + OFF_T preallocated_len = 0; |
5e3c6c93 WD |
125 | + |
126 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
127 | + /* Preallocate enough space for file's eventual length if | |
128 | + * possible; seems to reduce fragmentation on Windows. */ | |
129 | + if (posix_fallocate(fd, 0, total_size) == 0) | |
130 | + preallocated_len = total_size; | |
131 | + else | |
132 | + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); | |
133 | + } | |
134 | +#endif | |
135 | + | |
136 | read_sum_head(f_in, &sum); | |
137 | ||
138 | if (fd_r >= 0 && size_r > 0) { | |
cc3e685d | 139 | @@ -284,8 +298,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
5e3c6c93 WD |
140 | goto report_write_error; |
141 | ||
142 | #ifdef HAVE_FTRUNCATE | |
143 | - if (inplace && fd != -1) | |
144 | - ftruncate(fd, offset); | |
145 | + /* inplace: New data could be shorter than old data. | |
146 | + * preallocate_files: total_size could have been an overestimate. | |
147 | + * Cut off any extra preallocated zeros from dest file. */ | |
148 | + if ((inplace | |
149 | +#ifdef SUPPORT_PREALLOCATION | |
150 | + || preallocated_len > offset | |
151 | +#endif | |
152 | + ) && fd != -1) | |
153 | + if (ftruncate(fd, offset) < 0) | |
154 | + /* If we fail to truncate, the dest file may be wrong, so we | |
155 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 156 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(fname)); |
5e3c6c93 WD |
157 | #endif |
158 | ||
159 | if (do_progress) | |
cc3e685d WD |
160 | diff --git a/rsync.h b/rsync.h |
161 | --- a/rsync.h | |
162 | +++ b/rsync.h | |
a5e6228a | 163 | @@ -601,6 +601,10 @@ struct ht_int64_node { |
ffc18846 | 164 | #define ACLS_NEED_MASK 1 |
5e3c6c93 WD |
165 | #endif |
166 | ||
167 | +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE | |
168 | +#define SUPPORT_PREALLOCATION 1 | |
169 | +#endif | |
170 | + | |
612d3765 | 171 | union file_extras { |
c4bd76ea WD |
172 | int32 num; |
173 | uint32 unum; | |
cc3e685d WD |
174 | diff --git a/rsync.yo b/rsync.yo |
175 | --- a/rsync.yo | |
176 | +++ b/rsync.yo | |
177 | @@ -352,6 +352,7 @@ to the detailed description below for a complete description. verb( | |
612d3765 | 178 | --super receiver attempts super-user activities |
58b399b9 | 179 | --fake-super store/recover privileged attrs using xattrs |
612d3765 WD |
180 | -S, --sparse handle sparse files efficiently |
181 | + --preallocate posix_fallocate dest files before writing | |
e2b0842a | 182 | -n, --dry-run perform a trial run with no changes made |
f2863bc0 | 183 | -W, --whole-file copy files whole (w/o delta-xfer algorithm) |
612d3765 | 184 | -x, --one-file-system don't cross filesystem boundaries |
a5e6228a | 185 | @@ -1036,6 +1037,19 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" |
612d3765 WD |
186 | filesystem. It doesn't seem to handle seeks over null regions |
187 | correctly and ends up corrupting the files. | |
188 | ||
189 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
190 | +file to its eventual size using bf(posix_fallocate)(3) before writing data | |
191 | +to the file. If the receiver is remote, this nonstandard option only works | |
192 | +if the receiver also has the preallocation patch. Furthermore, this option | |
193 | +only works if the receiver found the bf(posix_fallocate)(3) call at | |
194 | +configure time. | |
195 | + | |
196 | +Without this option on MS Windows, very large destination files tend to be | |
197 | +broken into thousands of fragments; advising Windows ahead of time of the | |
198 | +eventual file size using this option usually reduces the number of | |
199 | +fragments to one. However, on Linux, this option appears to just waste | |
200 | +disk I/O. | |
201 | + | |
e2b0842a WD |
202 | dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't |
203 | make any changes (and produces mostly the same output as a real run). It | |
204 | is most commonly used in combination with the bf(-v, --verbose) and/or | |
cc3e685d WD |
205 | diff --git a/t_stub.c b/t_stub.c |
206 | --- a/t_stub.c | |
207 | +++ b/t_stub.c | |
ffc18846 | 208 | @@ -22,6 +22,7 @@ |
5e3c6c93 WD |
209 | #include "rsync.h" |
210 | ||
211 | int modify_window = 0; | |
212 | +int preallocate_files = 0; | |
213 | int module_id = -1; | |
214 | int relative_paths = 0; | |
215 | int human_readable = 0; | |
cc3e685d WD |
216 | diff --git a/util.c b/util.c |
217 | --- a/util.c | |
218 | +++ b/util.c | |
c8a8b4a7 | 219 | @@ -25,6 +25,7 @@ |
5e3c6c93 WD |
220 | |
221 | extern int verbose; | |
222 | extern int dry_run; | |
223 | +extern int preallocate_files; | |
224 | extern int module_id; | |
225 | extern int modify_window; | |
226 | extern int relative_paths; | |
4c15e800 | 227 | @@ -273,6 +274,10 @@ int copy_file(const char *source, const char *dest, int ofd, |
e2b0842a | 228 | int ifd; |
5e3c6c93 WD |
229 | char buf[1024 * 8]; |
230 | int len; /* Number of bytes read into `buf'. */ | |
231 | +#ifdef SUPPORT_PREALLOCATION | |
cc3e685d WD |
232 | + OFF_T preallocated_len = 0; |
233 | + OFF_T offset = 0; | |
5e3c6c93 WD |
234 | +#endif |
235 | ||
c8a8b4a7 | 236 | if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { |
cc3e685d | 237 | rsyserr(FERROR_XFER, errno, "open %s", full_fname(source)); |
4c15e800 | 238 | @@ -294,7 +299,27 @@ int copy_file(const char *source, const char *dest, int ofd, |
e2b0842a | 239 | } |
5e3c6c93 WD |
240 | } |
241 | ||
242 | +#ifdef SUPPORT_PREALLOCATION | |
243 | + if (preallocate_files) { | |
244 | + /* Preallocate enough space for file's eventual length if | |
245 | + * possible; seems to reduce fragmentation on Windows. */ | |
246 | + STRUCT_STAT srcst; | |
247 | + if (do_fstat(ifd, &srcst) == 0) { | |
248 | + if (srcst.st_size > 0) { | |
249 | + if (posix_fallocate(ofd, 0, srcst.st_size) == 0) | |
250 | + preallocated_len = srcst.st_size; | |
251 | + else | |
252 | + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); | |
253 | + } | |
254 | + } else | |
255 | + rsyserr(FINFO, errno, "fstat %s", full_fname(source)); | |
256 | + } | |
257 | +#endif | |
258 | + | |
259 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
260 | +#ifdef SUPPORT_PREALLOCATION | |
261 | + offset += len; | |
262 | +#endif | |
263 | if (full_write(ofd, buf, len) < 0) { | |
cc3e685d | 264 | rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest)); |
5e3c6c93 | 265 | close(ifd); |
4c15e800 | 266 | @@ -315,6 +340,16 @@ int copy_file(const char *source, const char *dest, int ofd, |
5e3c6c93 WD |
267 | full_fname(source)); |
268 | } | |
269 | ||
270 | +#ifdef SUPPORT_PREALLOCATION | |
271 | + /* Source file might have shrunk since we fstatted it. | |
272 | + * Cut off any extra preallocated zeros from dest file. */ | |
273 | + if (preallocated_len > offset) | |
274 | + if (ftruncate(ofd, offset) < 0) | |
275 | + /* If we fail to truncate, the dest file may be wrong, so we | |
276 | + * must trigger the "partial transfer" error. */ | |
cc3e685d | 277 | + rsyserr(FERROR_XFER, errno, "ftruncate %s", full_fname(dest)); |
5e3c6c93 WD |
278 | +#endif |
279 | + | |
280 | if (close(ofd) < 0) { | |
cc3e685d | 281 | rsyserr(FERROR_XFER, errno, "close failed on %s", |
5e3c6c93 | 282 | full_fname(dest)); |