| 1 | This patch adds the --preallocate option that asks rsync to preallocate the |
| 2 | copied files. This slows down the copy, but should reduce fragmentation on |
| 3 | systems that need that. |
| 4 | |
| 5 | To use this patch, run these commands for a successful build: |
| 6 | |
| 7 | patch -p1 <patches/preallocate.diff |
| 8 | ./prepare-source |
| 9 | ./configure |
| 10 | make |
| 11 | |
| 12 | --- old/configure.in |
| 13 | +++ new/configure.in |
| 14 | @@ -560,7 +560,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strd |
| 15 | strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ |
| 16 | setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ |
| 17 | strerror putenv iconv_open locale_charset nl_langinfo \ |
| 18 | - sigaction sigprocmask) |
| 19 | + sigaction sigprocmask posix_fallocate) |
| 20 | |
| 21 | AC_CHECK_FUNCS(getpgrp tcgetpgrp) |
| 22 | if test $ac_cv_func_getpgrp = yes; then |
| 23 | --- old/options.c |
| 24 | +++ new/options.c |
| 25 | @@ -69,6 +69,7 @@ int remove_source_files = 0; |
| 26 | int one_file_system = 0; |
| 27 | int protocol_version = PROTOCOL_VERSION; |
| 28 | int sparse_files = 0; |
| 29 | +int preallocate_files = 0; |
| 30 | int do_compression = 0; |
| 31 | int def_compress_level = Z_DEFAULT_COMPRESSION; |
| 32 | int am_root = 0; |
| 33 | @@ -202,6 +203,7 @@ static void print_rsync_version(enum log |
| 34 | char const *acls = "no "; |
| 35 | char const *links = "no "; |
| 36 | char const *ipv6 = "no "; |
| 37 | + char const *preallocation = "no "; |
| 38 | STRUCT_STAT *dumstat; |
| 39 | |
| 40 | #ifdef HAVE_SOCKETPAIR |
| 41 | @@ -228,6 +230,10 @@ static void print_rsync_version(enum log |
| 42 | ipv6 = ""; |
| 43 | #endif |
| 44 | |
| 45 | +#ifdef SUPPORT_PREALLOCATION |
| 46 | + preallocation = ""; |
| 47 | +#endif |
| 48 | + |
| 49 | rprintf(f, "%s version %s protocol version %d\n", |
| 50 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION); |
| 51 | rprintf(f, "Copyright (C) 1996-2007 by Andrew Tridgell, Wayne Davison, and others.\n"); |
| 52 | @@ -238,8 +244,8 @@ static void print_rsync_version(enum log |
| 53 | (int)(sizeof (int64) * 8)); |
| 54 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", |
| 55 | got_socketpair, hardlinks, links, ipv6, have_inplace); |
| 56 | - rprintf(f, " %sappend, %sACLs\n", |
| 57 | - have_inplace, acls); |
| 58 | + rprintf(f, " %sappend, %sACLs, %spreallocation\n", |
| 59 | + have_inplace, acls, preallocation); |
| 60 | |
| 61 | #ifdef MAINTAINER_MODE |
| 62 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); |
| 63 | @@ -319,6 +325,9 @@ void usage(enum logcode F) |
| 64 | rprintf(F," -O, --omit-dir-times omit directories when preserving times\n"); |
| 65 | rprintf(F," --super receiver attempts super-user activities\n"); |
| 66 | rprintf(F," -S, --sparse handle sparse files efficiently\n"); |
| 67 | +#ifdef SUPPORT_PREALLOCATION |
| 68 | + rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); |
| 69 | +#endif |
| 70 | rprintf(F," -n, --dry-run show what would have been transferred\n"); |
| 71 | rprintf(F," -W, --whole-file copy files whole (without rsync algorithm)\n"); |
| 72 | rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); |
| 73 | @@ -479,6 +488,7 @@ static struct poptOption long_options[] |
| 74 | {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, |
| 75 | {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, |
| 76 | {"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 }, |
| 77 | + {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, |
| 78 | {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, |
| 79 | {"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 }, |
| 80 | {"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 }, |
| 81 | @@ -1161,6 +1171,15 @@ int parse_arguments(int *argc, const cha |
| 82 | } |
| 83 | #endif |
| 84 | |
| 85 | +#ifndef SUPPORT_PREALLOCATION |
| 86 | + if (preallocate_files && !am_sender) { |
| 87 | + snprintf(err_buf, sizeof err_buf, |
| 88 | + "preallocation is not supported on this %s\n", |
| 89 | + am_server ? "server" : "client"); |
| 90 | + return 0; |
| 91 | + } |
| 92 | +#endif |
| 93 | + |
| 94 | if (write_batch && read_batch) { |
| 95 | snprintf(err_buf, sizeof err_buf, |
| 96 | "--write-batch and --read-batch can not be used together\n"); |
| 97 | @@ -1808,6 +1827,9 @@ void server_options(char **args,int *arg |
| 98 | else if (remove_source_files) |
| 99 | args[ac++] = "--remove-sent-files"; |
| 100 | |
| 101 | + if (preallocate_files && am_sender) |
| 102 | + args[ac++] = "--preallocate"; |
| 103 | + |
| 104 | *argc = ac; |
| 105 | return; |
| 106 | |
| 107 | --- old/receiver.c |
| 108 | +++ new/receiver.c |
| 109 | @@ -43,6 +43,7 @@ extern int cleanup_got_literal; |
| 110 | extern int remove_source_files; |
| 111 | extern int append_mode; |
| 112 | extern int sparse_files; |
| 113 | +extern int preallocate_files; |
| 114 | extern int keep_partial; |
| 115 | extern int checksum_seed; |
| 116 | extern int inplace; |
| 117 | @@ -136,6 +137,19 @@ static int receive_data(int f_in, char * |
| 118 | int32 i; |
| 119 | char *map = NULL; |
| 120 | |
| 121 | +#ifdef SUPPORT_PREALLOCATION |
| 122 | + int preallocated_len = 0; |
| 123 | + |
| 124 | + if (preallocate_files && fd != -1 && total_size > 0) { |
| 125 | + /* Preallocate enough space for file's eventual length if |
| 126 | + * possible; seems to reduce fragmentation on Windows. */ |
| 127 | + if (posix_fallocate(fd, 0, total_size) == 0) |
| 128 | + preallocated_len = total_size; |
| 129 | + else |
| 130 | + rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); |
| 131 | + } |
| 132 | +#endif |
| 133 | + |
| 134 | read_sum_head(f_in, &sum); |
| 135 | |
| 136 | if (fd_r >= 0 && size_r > 0) { |
| 137 | @@ -243,8 +257,18 @@ static int receive_data(int f_in, char * |
| 138 | goto report_write_error; |
| 139 | |
| 140 | #ifdef HAVE_FTRUNCATE |
| 141 | - if (inplace && fd != -1) |
| 142 | - ftruncate(fd, offset); |
| 143 | + /* inplace: New data could be shorter than old data. |
| 144 | + * preallocate_files: total_size could have been an overestimate. |
| 145 | + * Cut off any extra preallocated zeros from dest file. */ |
| 146 | + if ((inplace |
| 147 | +#ifdef SUPPORT_PREALLOCATION |
| 148 | + || preallocated_len > offset |
| 149 | +#endif |
| 150 | + ) && fd != -1) |
| 151 | + if (ftruncate(fd, offset) < 0) |
| 152 | + /* If we fail to truncate, the dest file may be wrong, so we |
| 153 | + * must trigger the "partial transfer" error. */ |
| 154 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(fname)); |
| 155 | #endif |
| 156 | |
| 157 | if (do_progress) |
| 158 | --- old/rsync.h |
| 159 | +++ new/rsync.h |
| 160 | @@ -551,6 +551,10 @@ struct idev_node { |
| 161 | #define ACLS_NEED_MASK 1 |
| 162 | #endif |
| 163 | |
| 164 | +#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE |
| 165 | +#define SUPPORT_PREALLOCATION 1 |
| 166 | +#endif |
| 167 | + |
| 168 | #define GID_NONE ((gid_t)-1) |
| 169 | |
| 170 | union file_extras { |
| 171 | --- old/rsync.yo |
| 172 | +++ new/rsync.yo |
| 173 | @@ -333,6 +333,7 @@ to the detailed description below for a |
| 174 | -O, --omit-dir-times omit directories when preserving times |
| 175 | --super receiver attempts super-user activities |
| 176 | -S, --sparse handle sparse files efficiently |
| 177 | + --preallocate posix_fallocate dest files before writing |
| 178 | -n, --dry-run show what would have been transferred |
| 179 | -W, --whole-file copy files whole (without rsync algorithm) |
| 180 | -x, --one-file-system don't cross filesystem boundaries |
| 181 | @@ -914,6 +915,19 @@ NOTE: Don't use this option when the des |
| 182 | filesystem. It doesn't seem to handle seeks over null regions |
| 183 | correctly and ends up corrupting the files. |
| 184 | |
| 185 | +dit(bf(--preallocate)) This tells the receiver to allocate each destination |
| 186 | +file to its eventual size using bf(posix_fallocate)(3) before writing data |
| 187 | +to the file. If the receiver is remote, this nonstandard option only works |
| 188 | +if the receiver also has the preallocation patch. Furthermore, this option |
| 189 | +only works if the receiver found the bf(posix_fallocate)(3) call at |
| 190 | +configure time. |
| 191 | + |
| 192 | +Without this option on MS Windows, very large destination files tend to be |
| 193 | +broken into thousands of fragments; advising Windows ahead of time of the |
| 194 | +eventual file size using this option usually reduces the number of |
| 195 | +fragments to one. However, on Linux, this option appears to just waste |
| 196 | +disk I/O. |
| 197 | + |
| 198 | dit(bf(-n, --dry-run)) This tells rsync to not do any file transfers, |
| 199 | instead it will just report the actions it would have taken. |
| 200 | |
| 201 | --- old/t_stub.c |
| 202 | +++ new/t_stub.c |
| 203 | @@ -22,6 +22,7 @@ |
| 204 | #include "rsync.h" |
| 205 | |
| 206 | int modify_window = 0; |
| 207 | +int preallocate_files = 0; |
| 208 | int module_id = -1; |
| 209 | int relative_paths = 0; |
| 210 | int human_readable = 0; |
| 211 | --- old/util.c |
| 212 | +++ new/util.c |
| 213 | @@ -24,6 +24,7 @@ |
| 214 | |
| 215 | extern int verbose; |
| 216 | extern int dry_run; |
| 217 | +extern int preallocate_files; |
| 218 | extern int module_id; |
| 219 | extern int modify_window; |
| 220 | extern int relative_paths; |
| 221 | @@ -269,6 +270,10 @@ int copy_file(const char *source, const |
| 222 | int ofd; |
| 223 | char buf[1024 * 8]; |
| 224 | int len; /* Number of bytes read into `buf'. */ |
| 225 | +#ifdef SUPPORT_PREALLOCATION |
| 226 | + int preallocated_len = 0; |
| 227 | + int offset = 0; |
| 228 | +#endif |
| 229 | |
| 230 | ifd = do_open(source, O_RDONLY, 0); |
| 231 | if (ifd == -1) { |
| 232 | @@ -288,7 +293,27 @@ int copy_file(const char *source, const |
| 233 | return -1; |
| 234 | } |
| 235 | |
| 236 | +#ifdef SUPPORT_PREALLOCATION |
| 237 | + if (preallocate_files) { |
| 238 | + /* Preallocate enough space for file's eventual length if |
| 239 | + * possible; seems to reduce fragmentation on Windows. */ |
| 240 | + STRUCT_STAT srcst; |
| 241 | + if (do_fstat(ifd, &srcst) == 0) { |
| 242 | + if (srcst.st_size > 0) { |
| 243 | + if (posix_fallocate(ofd, 0, srcst.st_size) == 0) |
| 244 | + preallocated_len = srcst.st_size; |
| 245 | + else |
| 246 | + rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); |
| 247 | + } |
| 248 | + } else |
| 249 | + rsyserr(FINFO, errno, "fstat %s", full_fname(source)); |
| 250 | + } |
| 251 | +#endif |
| 252 | + |
| 253 | while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { |
| 254 | +#ifdef SUPPORT_PREALLOCATION |
| 255 | + offset += len; |
| 256 | +#endif |
| 257 | if (full_write(ofd, buf, len) < 0) { |
| 258 | rsyserr(FERROR, errno, "write %s", full_fname(dest)); |
| 259 | close(ifd); |
| 260 | @@ -309,6 +334,16 @@ int copy_file(const char *source, const |
| 261 | full_fname(source)); |
| 262 | } |
| 263 | |
| 264 | +#ifdef SUPPORT_PREALLOCATION |
| 265 | + /* Source file might have shrunk since we fstatted it. |
| 266 | + * Cut off any extra preallocated zeros from dest file. */ |
| 267 | + if (preallocated_len > offset) |
| 268 | + if (ftruncate(ofd, offset) < 0) |
| 269 | + /* If we fail to truncate, the dest file may be wrong, so we |
| 270 | + * must trigger the "partial transfer" error. */ |
| 271 | + rsyserr(FERROR, errno, "ftruncate %s", full_fname(dest)); |
| 272 | +#endif |
| 273 | + |
| 274 | if (close(ofd) < 0) { |
| 275 | rsyserr(FERROR, errno, "close failed on %s", |
| 276 | full_fname(dest)); |