Commit | Line | Data |
---|---|---|
5e3c6c93 WD |
1 | This patch adds the --preallocate option that asks rsync to preallocate the |
2 | copied files. This slows down the copy, but should reduce fragmentation on | |
3 | systems that need that. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/preallocate.diff | |
8 | ./prepare-source | |
9 | ./configure | |
10 | make | |
11 | ||
5e3c6c93 WD |
12 | --- old/configure.in |
13 | +++ new/configure.in | |
612d3765 | 14 | @@ -555,7 +555,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd |
5e3c6c93 WD |
15 | strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ |
16 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ | |
17 | strerror putenv iconv_open locale_charset nl_langinfo \ | |
18 | - sigaction sigprocmask) | |
19 | + sigaction sigprocmask posix_fallocate) | |
20 | ||
21 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) | |
22 | if test $ac_cv_func_getpgrp = yes; then | |
23 | --- old/options.c | |
24 | +++ new/options.c | |
25 | @@ -69,6 +69,7 @@ int remove_source_files = 0; | |
26 | int one_file_system = 0; | |
27 | int protocol_version = PROTOCOL_VERSION; | |
28 | int sparse_files = 0; | |
29 | +int preallocate_files = 0; | |
30 | int do_compression = 0; | |
31 | int def_compress_level = Z_DEFAULT_COMPRESSION; | |
32 | int am_root = 0; | |
f2376a08 | 33 | @@ -201,6 +202,7 @@ static void print_rsync_version(enum log |
5e3c6c93 WD |
34 | char const *hardlinks = "no "; |
35 | char const *links = "no "; | |
36 | char const *ipv6 = "no "; | |
37 | + char const *preallocation = "no "; | |
38 | STRUCT_STAT *dumstat; | |
39 | ||
40 | #ifdef HAVE_SOCKETPAIR | |
f2376a08 | 41 | @@ -223,6 +225,10 @@ static void print_rsync_version(enum log |
5e3c6c93 WD |
42 | ipv6 = ""; |
43 | #endif | |
44 | ||
45 | +#ifdef SUPPORT_PREALLOCATION | |
46 | + preallocation = ""; | |
47 | +#endif | |
48 | + | |
49 | rprintf(f, "%s version %s protocol version %d\n", | |
50 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION); | |
51 | rprintf(f, "Copyright (C) 1996-2006 by Andrew Tridgell, Wayne Davison, and others.\n"); | |
f2376a08 | 52 | @@ -233,8 +239,8 @@ static void print_rsync_version(enum log |
5e3c6c93 WD |
53 | (int)(sizeof (int64) * 8)); |
54 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
55 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
56 | - rprintf(f, " %sappend\n", | |
57 | - have_inplace); | |
58 | + rprintf(f, " %sappend, %spreallocation\n", | |
59 | + have_inplace, preallocation); | |
60 | ||
61 | #ifdef MAINTAINER_MODE | |
62 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
f2376a08 | 63 | @@ -311,6 +317,9 @@ void usage(enum logcode F) |
5e3c6c93 WD |
64 | rprintf(F," -O, --omit-dir-times omit directories when preserving times\n"); |
65 | rprintf(F," --super receiver attempts super-user activities\n"); | |
66 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); | |
67 | +#ifdef SUPPORT_PREALLOCATION | |
68 | + rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); | |
69 | +#endif | |
70 | rprintf(F," -n, --dry-run show what would have been transferred\n"); | |
71 | rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n"); | |
72 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); | |
f2376a08 | 73 | @@ -468,6 +477,7 @@ static struct poptOption long_options[] |
5e3c6c93 WD |
74 | {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, |
75 | {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, | |
76 | {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, | |
77 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, | |
78 | {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, | |
79 | {"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 }, | |
80 | {"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 }, | |
f2376a08 | 81 | @@ -1132,6 +1142,15 @@ int parse_arguments(int *argc, const cha |
5e3c6c93 WD |
82 | } |
83 | #endif | |
84 | ||
85 | +#ifndef SUPPORT_PREALLOCATION | |
86 | + if (preallocate_files && !am_sender) { | |
87 | + snprintf(err_buf, sizeof err_buf, | |
88 | + "preallocation is not supported on this %s\n", | |
89 | + am_server ? "server" : "client"); | |
90 | + return 0; | |
91 | + } | |
92 | +#endif | |
93 | + | |
94 | if (write_batch && read_batch) { | |
95 | snprintf(err_buf, sizeof err_buf, | |
96 | "--write-batch and --read-batch can not be used together\n"); | |
f2376a08 | 97 | @@ -1779,6 +1798,9 @@ void server_options(char **args,int *arg |
5e3c6c93 WD |
98 | else if (remove_source_files) |
99 | args[ac++] = "--remove-sent-files"; | |
100 | ||
101 | + if (preallocate_files && am_sender) | |
102 | + args[ac++] = "--preallocate"; | |
103 | + | |
104 | *argc = ac; | |
105 | return; | |
106 | ||
107 | --- old/receiver.c | |
108 | +++ new/receiver.c | |
f2376a08 | 109 | @@ -44,6 +44,7 @@ extern int cleanup_got_literal; |
5e3c6c93 WD |
110 | extern int remove_source_files; |
111 | extern int append_mode; | |
112 | extern int sparse_files; | |
113 | +extern int preallocate_files; | |
114 | extern int keep_partial; | |
115 | extern int checksum_seed; | |
116 | extern int inplace; | |
117 | @@ -137,6 +138,19 @@ static int receive_data(int f_in, char * | |
118 | int32 i; | |
119 | char *map = NULL; | |
120 | ||
121 | +#ifdef SUPPORT_PREALLOCATION | |
122 | + int preallocated_len = 0; | |
123 | + | |
124 | + if (preallocate_files && fd != -1 && total_size > 0) { | |
125 | + /* Preallocate enough space for file's eventual length if | |
126 | + * possible; seems to reduce fragmentation on Windows. */ | |
127 | + if (posix_fallocate(fd, 0, total_size) == 0) | |
128 | + preallocated_len = total_size; | |
129 | + else | |
130 | + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); | |
131 | + } | |
132 | +#endif | |
133 | + | |
134 | read_sum_head(f_in, &sum); | |
135 | ||
136 | if (fd_r >= 0 && size_r > 0) { | |
137 | @@ -244,8 +258,18 @@ static int receive_data(int f_in, char * | |
138 | goto report_write_error; | |
139 | ||
140 | #ifdef HAVE_FTRUNCATE | |
141 | - if (inplace && fd != -1) | |
142 | - ftruncate(fd, offset); | |
143 | + /* inplace: New data could be shorter than old data. | |
144 | + * preallocate_files: total_size could have been an overestimate. | |
145 | + * Cut off any extra preallocated zeros from dest file. */ | |
146 | + if ((inplace | |
147 | +#ifdef SUPPORT_PREALLOCATION | |
148 | + || preallocated_len > offset | |
149 | +#endif | |
150 | + ) && fd != -1) | |
151 | + if (ftruncate(fd, offset) < 0) | |
152 | + /* If we fail to truncate, the dest file may be wrong, so we | |
153 | + * must trigger the "partial transfer" error. */ | |
154 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname)); | |
155 | #endif | |
156 | ||
157 | if (do_progress) | |
158 | --- old/rsync.h | |
159 | +++ new/rsync.h | |
f2376a08 | 160 | @@ -546,6 +546,10 @@ struct idev_node { |
5e3c6c93 WD |
161 | #define IN_LOOPBACKNET 127 |
162 | #endif | |
163 | ||
164 | +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE | |
165 | +#define SUPPORT_PREALLOCATION 1 | |
166 | +#endif | |
167 | + | |
168 | #define GID_NONE ((gid_t)-1) | |
169 | ||
612d3765 WD |
170 | union file_extras { |
171 | --- old/rsync.yo | |
172 | +++ new/rsync.yo | |
173 | @@ -332,6 +332,7 @@ to the detailed description below for a | |
174 | -O, --omit-dir-times omit directories when preserving times | |
175 | --super receiver attempts super-user activities | |
176 | -S, --sparse handle sparse files efficiently | |
177 | + --preallocate posix_fallocate dest files before writing | |
178 | -n, --dry-run show what would have been transferred | |
179 | -W, --whole-file copy files whole (without rsync algorithm) | |
180 | -x, --one-file-system don't cross filesystem boundaries | |
181 | @@ -888,6 +889,19 @@ NOTE: Don't use this option when the des | |
182 | filesystem. It doesn't seem to handle seeks over null regions | |
183 | correctly and ends up corrupting the files. | |
184 | ||
185 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination | |
186 | +file to its eventual size using bf(posix_fallocate)(3) before writing data | |
187 | +to the file. If the receiver is remote, this nonstandard option only works | |
188 | +if the receiver also has the preallocation patch. Furthermore, this option | |
189 | +only works if the receiver found the bf(posix_fallocate)(3) call at | |
190 | +configure time. | |
191 | + | |
192 | +Without this option on MS Windows, very large destination files tend to be | |
193 | +broken into thousands of fragments; advising Windows ahead of time of the | |
194 | +eventual file size using this option usually reduces the number of | |
195 | +fragments to one. However, on Linux, this option appears to just waste | |
196 | +disk I/O. | |
197 | + | |
198 | dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers, | |
199 | instead it will just report the actions it would have taken. | |
200 | ||
5e3c6c93 WD |
201 | --- old/t_stub.c |
202 | +++ new/t_stub.c | |
203 | @@ -23,6 +23,7 @@ | |
204 | #include "rsync.h" | |
205 | ||
206 | int modify_window = 0; | |
207 | +int preallocate_files = 0; | |
208 | int module_id = -1; | |
209 | int relative_paths = 0; | |
210 | int human_readable = 0; | |
211 | --- old/util.c | |
212 | +++ new/util.c | |
213 | @@ -25,6 +25,7 @@ | |
214 | ||
215 | extern int verbose; | |
216 | extern int dry_run; | |
217 | +extern int preallocate_files; | |
218 | extern int module_id; | |
219 | extern int modify_window; | |
220 | extern int relative_paths; | |
221 | @@ -270,6 +271,10 @@ int copy_file(const char *source, const | |
222 | int ofd; | |
223 | char buf[1024 * 8]; | |
224 | int len; /* Number of bytes read into `buf'. */ | |
225 | +#ifdef SUPPORT_PREALLOCATION | |
226 | + int preallocated_len = 0; | |
227 | + int offset = 0; | |
228 | +#endif | |
229 | ||
230 | ifd = do_open(source, O_RDONLY, 0); | |
231 | if (ifd == -1) { | |
232 | @@ -289,7 +294,27 @@ int copy_file(const char *source, const | |
233 | return -1; | |
234 | } | |
235 | ||
236 | +#ifdef SUPPORT_PREALLOCATION | |
237 | + if (preallocate_files) { | |
238 | + /* Preallocate enough space for file's eventual length if | |
239 | + * possible; seems to reduce fragmentation on Windows. */ | |
240 | + STRUCT_STAT srcst; | |
241 | + if (do_fstat(ifd, &srcst) == 0) { | |
242 | + if (srcst.st_size > 0) { | |
243 | + if (posix_fallocate(ofd, 0, srcst.st_size) == 0) | |
244 | + preallocated_len = srcst.st_size; | |
245 | + else | |
246 | + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); | |
247 | + } | |
248 | + } else | |
249 | + rsyserr(FINFO, errno, "fstat %s", full_fname(source)); | |
250 | + } | |
251 | +#endif | |
252 | + | |
253 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { | |
254 | +#ifdef SUPPORT_PREALLOCATION | |
255 | + offset += len; | |
256 | +#endif | |
257 | if (full_write(ofd, buf, len) < 0) { | |
258 | rsyserr(FERROR, errno, "write %s", full_fname(dest)); | |
259 | close(ifd); | |
260 | @@ -310,6 +335,16 @@ int copy_file(const char *source, const | |
261 | full_fname(source)); | |
262 | } | |
263 | ||
264 | +#ifdef SUPPORT_PREALLOCATION | |
265 | + /* Source file might have shrunk since we fstatted it. | |
266 | + * Cut off any extra preallocated zeros from dest file. */ | |
267 | + if (preallocated_len > offset) | |
268 | + if (ftruncate(ofd, offset) < 0) | |
269 | + /* If we fail to truncate, the dest file may be wrong, so we | |
270 | + * must trigger the "partial transfer" error. */ | |
271 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest)); | |
272 | +#endif | |
273 | + | |
274 | if (close(ofd) < 0) { | |
275 | rsyserr(FERROR, errno, "close failed on %s", | |
276 | full_fname(dest)); |