Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
5e3c6c93 WD |
12 | --- old/configure.in |
13 | +++ new/configure.in | |
82c2230a | 14 | @@ -561,7 +561,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd |
5e3c6c93 WD |
15 | strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ |
16 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ | |
80c89075 WD |
17 | strerror putenv iconv_open locale_charset nl_langinfo getxattr \ |
18 | - extattr_get_link sigaction sigprocmask) | |
19 | + extattr_get_link sigaction sigprocmask posix_fallocate) | |
5e3c6c93 WD |
20 | |
21 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) | |
22 | if test $ac_cv_func_getpgrp = yes; then | |
23 | --- old/options.c | |
24 | +++ new/options.c | |
5795bf59 | 25 | @@ -70,6 +70,7 @@ int remove_source_files = 0; |
5e3c6c93 WD |
26 | int one_file_system = 0; |
27 | int protocol_version = PROTOCOL_VERSION; | |
28 | int sparse_files = 0; | |
29 | +int preallocate_files = 0; | |
30 | int do_compression = 0; | |
31 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
58b399b9 | 32 | int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ |
790ba11a | 33 | @@ -220,6 +221,7 @@ static void print_rsync_version(enum log |
5e3c6c93 | 34 | char const *links = "no "; |
58b399b9 | 35 | char const *iconv = "no "; |
5e3c6c93 WD |
36 | char const *ipv6 = "no "; |
37 | + char const *preallocation = "no "; | |
38 | STRUCT_STAT *dumstat; | |
39 | ||
ac2da598 | 40 | #if SUBPROTOCOL_VERSION != 0 |
790ba11a | 41 | @@ -249,6 +251,9 @@ static void print_rsync_version(enum log |
58b399b9 WD |
42 | #ifdef ICONV_OPTION |
43 | iconv = ""; | |
5e3c6c93 | 44 | #endif |
5e3c6c93 WD |
45 | +#ifdef SUPPORT_PREALLOCATION |
46 | + preallocation = ""; | |
47 | +#endif | |
ac2da598 WD |
48 | |
49 | rprintf(f, "%s version %s protocol version %d%s\n", | |
50 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
790ba11a | 51 | @@ -262,8 +267,8 @@ static void print_rsync_version(enum log |
5e3c6c93 WD |
52 | (int)(sizeof (int64) * 8)); |
53 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
54 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
58b399b9 WD |
55 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv\n", |
56 | - have_inplace, acls, xattrs, iconv); | |
57 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %spreallocation\n", | |
58 | + have_inplace, acls, xattrs, iconv, preallocation); | |
5e3c6c93 WD |
59 | |
60 | #ifdef MAINTAINER_MODE | |
61 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
790ba11a | 62 | @@ -350,6 +355,9 @@ void usage(enum logcode F) |
58b399b9 WD |
63 | rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); |
64 | #endif | |
5e3c6c93 WD |
65 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
66 | +#ifdef SUPPORT_PREALLOCATION | |
67 | + rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); | |
68 | +#endif | |
69 | rprintf(F," -n, --dry-run show what would have been transferred\n"); | |
70 | rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n"); | |
71 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); | |
790ba11a | 72 | @@ -525,6 +533,7 @@ static struct poptOption long_options[] |
5e3c6c93 WD |
73 | {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, |
74 | {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, | |
75 | {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, | |
76 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, | |
77 | {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, | |
790ba11a WD |
78 | {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, |
79 | {"append-verify", 0, POPT_ARG_VAL, &append_mode, 2, 0, 0 }, | |
80 | @@ -1279,6 +1288,15 @@ int parse_arguments(int *argc_p, const c | |
5e3c6c93 WD |
81 | } |
82 | #endif | |
83 | ||
84 | +#ifndef SUPPORT_PREALLOCATION | |
85 | + if (preallocate_files && !am_sender) { | |
86 | + snprintf(err_buf, sizeof err_buf, | |
87 | + "preallocation is not supported on this %s\n", | |
88 | + am_server ? "server" : "client"); | |
89 | + return 0; | |
90 | + } | |
91 | +#endif | |
92 | + | |
93 | if (write_batch && read_batch) { | |
94 | snprintf(err_buf, sizeof err_buf, | |
95 | "--write-batch and --read-batch can not be used together\n"); | |
790ba11a | 96 | @@ -1978,6 +1996,9 @@ void server_options(char **args, int *ar |
5e3c6c93 WD |
97 | else if (remove_source_files) |
98 | args[ac++] = "--remove-sent-files"; | |
99 | ||
100 | + if (preallocate_files && am_sender) | |
101 | + args[ac++] = "--preallocate"; | |
102 | + | |
790ba11a | 103 | *argc_p = ac; |
5e3c6c93 WD |
104 | return; |
105 | ||
106 | --- old/receiver.c | |
107 | +++ new/receiver.c | |
5795bf59 | 108 | @@ -45,6 +45,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
109 | extern int remove_source_files; |
110 | extern int append_mode; | |
111 | extern int sparse_files; | |
112 | +extern int preallocate_files; | |
113 | extern int keep_partial; | |
114 | extern int checksum_seed; | |
115 | extern int inplace; | |
5795bf59 | 116 | @@ -138,6 +139,19 @@ static int receive_data(int f_in, char * |
5e3c6c93 WD |
117 | int32 i; |
118 | char *map = NULL; | |
119 | ||
120 | +#ifdef SUPPORT_PREALLOCATION | |
121 | + int preallocated_len = 0; | |
122 | + | |
123 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
124 | + /* Preallocate enough space for file's eventual length if | |
125 | + * possible; seems to reduce fragmentation on Windows. */ | |
126 | + if (posix_fallocate(fd, 0, total_size) == 0) | |
127 | + preallocated_len = total_size; | |
128 | + else | |
129 | + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); | |
130 | + } | |
131 | +#endif | |
132 | + | |
133 | read_sum_head(f_in, &sum); | |
134 | ||
135 | if (fd_r >= 0 && size_r > 0) { | |
790ba11a | 136 | @@ -247,8 +261,18 @@ static int receive_data(int f_in, char * |
5e3c6c93 WD |
137 | goto report_write_error; |
138 | ||
139 | #ifdef HAVE_FTRUNCATE | |
140 | - if (inplace && fd != -1) | |
141 | - ftruncate(fd, offset); | |
142 | + /* inplace: New data could be shorter than old data. | |
143 | + * preallocate_files: total_size could have been an overestimate. | |
144 | + * Cut off any extra preallocated zeros from dest file. */ | |
145 | + if ((inplace | |
146 | +#ifdef SUPPORT_PREALLOCATION | |
147 | + || preallocated_len > offset | |
148 | +#endif | |
149 | + ) && fd != -1) | |
150 | + if (ftruncate(fd, offset) < 0) | |
151 | + /* If we fail to truncate, the dest file may be wrong, so we | |
152 | + * must trigger the "partial transfer" error. */ | |
153 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname)); | |
154 | #endif | |
155 | ||
156 | if (do_progress) | |
157 | --- old/rsync.h | |
158 | +++ new/rsync.h | |
790ba11a | 159 | @@ -555,6 +555,10 @@ struct ht_int64_node { |
ffc18846 | 160 | #define ACLS_NEED_MASK 1 |
5e3c6c93 WD |
161 | #endif |
162 | ||
163 | +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE | |
164 | +#define SUPPORT_PREALLOCATION 1 | |
165 | +#endif | |
166 | + | |
612d3765 | 167 | union file_extras { |
c4bd76ea WD |
168 | int32 num; |
169 | uint32 unum; | |
612d3765 WD |
170 | --- old/rsync.yo |
171 | +++ new/rsync.yo | |
790ba11a | 172 | @@ -357,6 +357,7 @@ to the detailed description below for a |
612d3765 | 173 | --super receiver attempts super-user activities |
58b399b9 | 174 | --fake-super store/recover privileged attrs using xattrs |
612d3765 WD |
175 | -S, --sparse handle sparse files efficiently |
176 | + --preallocate posix_fallocate dest files before writing | |
177 | -n, --dry-run show what would have been transferred | |
178 | -W, --whole-file copy files whole (without rsync algorithm) | |
179 | -x, --one-file-system don't cross filesystem boundaries | |
790ba11a | 180 | @@ -993,6 +994,19 @@ NOTE: Don't use this option when the des |
612d3765 WD |
181 | filesystem. It doesn't seem to handle seeks over null regions |
182 | correctly and ends up corrupting the files. | |
183 | ||
184 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
185 | +file to its eventual size using bf(posix_fallocate)(3) before writing data | |
186 | +to the file. If the receiver is remote, this nonstandard option only works | |
187 | +if the receiver also has the preallocation patch. Furthermore, this option | |
188 | +only works if the receiver found the bf(posix_fallocate)(3) call at | |
189 | +configure time. | |
190 | + | |
191 | +Without this option on MS Windows, very large destination files tend to be | |
192 | +broken into thousands of fragments; advising Windows ahead of time of the | |
193 | +eventual file size using this option usually reduces the number of | |
194 | +fragments to one. However, on Linux, this option appears to just waste | |
195 | +disk I/O. | |
196 | + | |
197 | dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers, | |
198 | instead it will just report the actions it would have taken. | |
199 | ||
5e3c6c93 WD |
200 | --- old/t_stub.c |
201 | +++ new/t_stub.c | |
ffc18846 | 202 | @@ -22,6 +22,7 @@ |
5e3c6c93 WD |
203 | #include "rsync.h" |
204 | ||
205 | int modify_window = 0; | |
206 | +int preallocate_files = 0; | |
207 | int module_id = -1; | |
208 | int relative_paths = 0; | |
209 | int human_readable = 0; | |
210 | --- old/util.c | |
211 | +++ new/util.c | |
ffc18846 | 212 | @@ -24,6 +24,7 @@ |
5e3c6c93 WD |
213 | |
214 | extern int verbose; | |
215 | extern int dry_run; | |
216 | +extern int preallocate_files; | |
217 | extern int module_id; | |
218 | extern int modify_window; | |
219 | extern int relative_paths; | |
790ba11a | 220 | @@ -270,6 +271,10 @@ int copy_file(const char *source, const |
5e3c6c93 WD |
221 | int ofd; |
222 | char buf[1024 * 8]; | |
223 | int len; /* Number of bytes read into `buf'. */ | |
224 | +#ifdef SUPPORT_PREALLOCATION | |
225 | + int preallocated_len = 0; | |
226 | + int offset = 0; | |
227 | +#endif | |
228 | ||
229 | ifd = do_open(source, O_RDONLY, 0); | |
230 | if (ifd == -1) { | |
790ba11a | 231 | @@ -289,7 +294,27 @@ int copy_file(const char *source, const |
5e3c6c93 WD |
232 | return -1; |
233 | } | |
234 | ||
235 | +#ifdef SUPPORT_PREALLOCATION | |
236 | + if (preallocate_files) { | |
237 | + /* Preallocate enough space for file's eventual length if | |
238 | + * possible; seems to reduce fragmentation on Windows. */ | |
239 | + STRUCT_STAT srcst; | |
240 | + if (do_fstat(ifd, &srcst) == 0) { | |
241 | + if (srcst.st_size > 0) { | |
242 | + if (posix_fallocate(ofd, 0, srcst.st_size) == 0) | |
243 | + preallocated_len = srcst.st_size; | |
244 | + else | |
245 | + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); | |
246 | + } | |
247 | + } else | |
248 | + rsyserr(FINFO, errno, "fstat %s", full_fname(source)); | |
249 | + } | |
250 | +#endif | |
251 | + | |
252 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
253 | +#ifdef SUPPORT_PREALLOCATION | |
254 | + offset += len; | |
255 | +#endif | |
256 | if (full_write(ofd, buf, len) < 0) { | |
257 | rsyserr(FERROR, errno, "write %s", full_fname(dest)); | |
258 | close(ifd); | |
790ba11a | 259 | @@ -310,6 +335,16 @@ int copy_file(const char *source, const |
5e3c6c93 WD |
260 | full_fname(source)); |
261 | } | |
262 | ||
263 | +#ifdef SUPPORT_PREALLOCATION | |
264 | + /* Source file might have shrunk since we fstatted it. | |
265 | + * Cut off any extra preallocated zeros from dest file. */ | |
266 | + if (preallocated_len > offset) | |
267 | + if (ftruncate(ofd, offset) < 0) | |
268 | + /* If we fail to truncate, the dest file may be wrong, so we | |
269 | + * must trigger the "partial transfer" error. */ | |
270 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest)); | |
271 | +#endif | |
272 | + | |
273 | if (close(ofd) < 0) { | |
274 | rsyserr(FERROR, errno, "close failed on %s", | |
275 | full_fname(dest)); |