Commit | Line | Data |
---|---|---|
ae10e51e WD |
1 | This adds a sender optimization feature that allows a cache of checksums |
2 | to be created/updated and used when the client specifies the --checksum | |
3 | option. | |
4 | ||
5 | To use this patch, run these commands for a successful build: | |
6 | ||
7 | patch -p1 <patches/checksum-updating.diff | |
8 | ./configure (optional if already run) | |
9 | make | |
10 | ||
11 | TODO: when sending individual files (as opposed to an entire directory), | |
12 | we should still update the .md[45]sums file(s) if we compute new checksum | |
13 | info. (Writing currently only occurs if we send an entire dir.) | |
14 | ||
15 | --- old/clientserver.c | |
16 | +++ new/clientserver.c | |
17 | @@ -37,6 +37,7 @@ extern int sanitize_paths; | |
18 | extern int filesfrom_fd; | |
19 | extern int remote_protocol; | |
20 | extern int protocol_version; | |
21 | +extern int checksum_updating; | |
22 | extern int io_timeout; | |
23 | extern int no_detach; | |
24 | extern int default_af_hint; | |
25 | @@ -634,6 +635,8 @@ static int rsync_module(int f_in, int f_ | |
26 | else if (am_root < 0) /* Treat --fake-super from client as --super. */ | |
27 | am_root = 2; | |
28 | ||
29 | + checksum_updating = lp_checksum_updating(i); | |
30 | + | |
31 | if (filesfrom_fd == 0) | |
32 | filesfrom_fd = f_in; | |
33 | ||
34 | --- old/flist.c | |
35 | +++ new/flist.c | |
36 | @@ -25,6 +25,7 @@ | |
37 | #include "io.h" | |
38 | ||
39 | extern int verbose; | |
40 | +extern int dry_run; | |
41 | extern int list_only; | |
42 | extern int am_root; | |
43 | extern int am_server; | |
44 | @@ -57,6 +58,7 @@ extern int implied_dirs; | |
45 | extern int file_extra_cnt; | |
46 | extern int ignore_perishable; | |
47 | extern int non_perishable_cnt; | |
48 | +extern int checksum_updating; | |
49 | extern int prune_empty_dirs; | |
50 | extern int copy_links; | |
51 | extern int copy_unsafe_links; | |
52 | @@ -101,6 +103,8 @@ static char tmp_sum[MAX_DIGEST_LEN]; | |
53 | static char empty_sum[MAX_DIGEST_LEN]; | |
54 | static int flist_count_offset; /* for --delete --progress */ | |
55 | static int dir_count = 0; | |
56 | +static struct file_list *checksum_flist = NULL; | |
57 | +static int checksum_matches = 0; | |
58 | ||
59 | static void clean_flist(struct file_list *flist, int strip_root); | |
60 | static void output_flist(struct file_list *flist); | |
61 | @@ -317,6 +321,216 @@ static void flist_done_allocating(struct | |
62 | flist->pool_boundary = ptr; | |
63 | } | |
64 | ||
65 | +/* The len count is the length of the basename + 1 for the null. */ | |
66 | +static void add_checksum(const char *dirname, const char *basename, int len, | |
67 | + OFF_T file_length, time_t mtime, const char *sum, | |
68 | + int flags) | |
69 | +{ | |
70 | + struct file_struct *file; | |
71 | + int alloc_len, extra_len; | |
72 | + char *bp; | |
73 | + | |
74 | + if (len == 8+1 && *basename == '.' | |
75 | + && (strcmp(basename, ".md5sums") == 0 | |
76 | + || strcmp(basename, ".md4sums") == 0)) | |
77 | + return; | |
78 | + | |
79 | + if (len < 0) | |
80 | + len = strlen(basename) + 1; | |
81 | + | |
82 | + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT) | |
83 | + * EXTRA_LEN; | |
84 | +#if EXTRA_ROUNDING > 0 | |
85 | + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) | |
86 | + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; | |
87 | +#endif | |
88 | + alloc_len = FILE_STRUCT_LEN + extra_len + len; | |
89 | + bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); | |
90 | + | |
91 | + memset(bp, 0, extra_len + FILE_STRUCT_LEN); | |
92 | + bp += extra_len; | |
93 | + file = (struct file_struct *)bp; | |
94 | + bp += FILE_STRUCT_LEN; | |
95 | + | |
96 | + memcpy(bp, basename, len); | |
97 | + | |
98 | + file->flags = flags; | |
99 | + file->mode = S_IFREG; | |
100 | + file->modtime = mtime; | |
101 | + file->len32 = (uint32)file_length; | |
102 | + if (file_length > 0xFFFFFFFFu) { | |
103 | + file->flags |= FLAG_LENGTH64; | |
104 | + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); | |
105 | + } | |
106 | + file->dirname = dirname; | |
107 | + bp = (char*)F_SUM(file); | |
108 | + memcpy(bp, sum, checksum_len); | |
109 | + | |
110 | + flist_expand(checksum_flist, 1); | |
111 | + checksum_flist->files[checksum_flist->count++] = file; | |
112 | + | |
113 | + checksum_flist->sorted = checksum_flist->files; | |
114 | +} | |
115 | + | |
116 | +/* The direname value must remain unchanged during the lifespan of the | |
117 | + * created checksum_flist object because we use it directly. */ | |
118 | +static void read_checksums(const char *dirname) | |
119 | +{ | |
120 | + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; | |
121 | + const char *filename; | |
122 | + OFF_T file_length; | |
123 | + time_t mtime; | |
124 | + int len, dlen, i, flags; | |
125 | + char *cp; | |
126 | + FILE *fp; | |
127 | + | |
128 | + if (checksum_flist) { | |
129 | + /* Reset the pool memory and empty the file-list array. */ | |
130 | + pool_free_old(checksum_flist->file_pool, | |
131 | + pool_boundary(checksum_flist->file_pool, 0)); | |
132 | + checksum_flist->count = 0; | |
133 | + } else | |
134 | + checksum_flist = flist_new(FLIST_TEMP, "read_checksums"); | |
135 | + | |
136 | + checksum_flist->low = 0; | |
137 | + checksum_flist->high = -1; | |
138 | + checksum_matches = 0; | |
139 | + | |
140 | + if (protocol_version >= 30) | |
141 | + filename = ".md5sums"; | |
142 | + else | |
143 | + filename = ".md4sums"; | |
144 | + if (dirname) { | |
145 | + dlen = strlcpy(fbuf, dirname, sizeof fbuf); | |
146 | + if (dlen >= (int)sizeof fbuf) | |
147 | + return; | |
148 | + fbuf[dlen++] = '/'; | |
149 | + } else | |
150 | + dlen = 0; | |
151 | + strlcpy(fbuf+dlen, filename, sizeof fbuf - dlen); | |
152 | + if (!(fp = fopen(fbuf, "r"))) | |
153 | + return; | |
154 | + | |
155 | + while (fgets(line, sizeof line, fp)) { | |
156 | + for (i = 0, cp = line; i < checksum_len*2; i++, cp++) { | |
157 | + int x; | |
158 | + if (isDigit(cp)) | |
159 | + x = *cp - '0'; | |
160 | + else if (isAlpha(cp)) { | |
161 | + x = (*cp & 0xF) + 9; | |
162 | + if (x > 0xF) { | |
163 | + cp = ""; | |
164 | + break; | |
165 | + } | |
166 | + } else { | |
167 | + cp = ""; | |
168 | + break; | |
169 | + } | |
170 | + if (i & 1) | |
171 | + sum[i/2] |= x; | |
172 | + else | |
173 | + sum[i/2] = x << 4; | |
174 | + } | |
175 | + | |
176 | + if (*cp != ' ') | |
177 | + continue; | |
178 | + while (*++cp == ' ') {} | |
179 | + | |
180 | + file_length = 0; | |
181 | + while (isDigit(cp)) | |
182 | + file_length = file_length * 10 + *cp++ - '0'; | |
183 | + | |
184 | + if (*cp != ' ') | |
185 | + continue; | |
186 | + while (*++cp == ' ') {} | |
187 | + | |
188 | + mtime = 0; | |
189 | + while (isDigit(cp)) | |
190 | + mtime = mtime * 10 + *cp++ - '0'; | |
191 | + | |
192 | + if (*cp != ' ') | |
193 | + continue; | |
194 | + while (*++cp == ' ') {} | |
195 | + | |
196 | + len = strlen(cp); | |
197 | + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r')) | |
198 | + len--; | |
199 | + if (!len) | |
200 | + continue; | |
201 | + cp[len++] = '\0'; /* len now counts the null */ | |
202 | + if (strchr(cp, '/') || len > MAXPATHLEN) | |
203 | + continue; | |
204 | + | |
205 | + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); | |
206 | + if (is_excluded(fbuf, 0, ALL_FILTERS)) { | |
207 | + flags = FLAG_FILE_SENT; | |
208 | + checksum_matches++; | |
209 | + } else | |
210 | + flags = 0; | |
211 | + | |
212 | + add_checksum(dirname, cp, len, file_length, mtime, sum, flags); | |
213 | + } | |
214 | + fclose(fp); | |
215 | + | |
216 | + clean_flist(checksum_flist, 0); | |
217 | +} | |
218 | + | |
219 | +static void write_checksums(const char *dirname) | |
220 | +{ | |
221 | + char buf[MAXPATHLEN+1024]; | |
222 | + const char *filename; | |
223 | + int new_entries = checksum_flist->count > checksum_flist->high + 1; | |
224 | + int orphan_entires = checksum_flist->count != checksum_matches; | |
225 | + FILE *out_fp; | |
226 | + int i; | |
227 | + | |
228 | + if (dry_run) | |
229 | + return; | |
230 | + | |
231 | + for (i = checksum_flist->high + 1; i < checksum_flist->count; i++) { | |
232 | + struct file_struct *file = checksum_flist->sorted[i]; | |
233 | + file->flags |= FLAG_FILE_SENT; | |
234 | + } | |
235 | + | |
236 | + clean_flist(checksum_flist, 0); | |
237 | + | |
238 | + if (protocol_version >= 30) | |
239 | + filename = ".md5sums"; | |
240 | + else | |
241 | + filename = ".md4sums"; | |
242 | + if (dirname) { | |
243 | + if (pathjoin(buf, sizeof buf, dirname, filename) >= sizeof buf) | |
244 | + return; | |
245 | + } else | |
246 | + strlcpy(buf, filename, sizeof buf); | |
247 | + | |
248 | + if (checksum_flist->high - checksum_flist->low < 0) { | |
249 | + unlink(buf); | |
250 | + return; | |
251 | + } | |
252 | + | |
253 | + if (!new_entries && !orphan_entires) | |
254 | + return; | |
255 | + | |
256 | + if (!(out_fp = fopen(buf, "w"))) | |
257 | + return; | |
258 | + | |
259 | + for (i = checksum_flist->low; i <= checksum_flist->high; i++) { | |
260 | + struct file_struct *file = checksum_flist->sorted[i]; | |
261 | + const char *cp = F_SUM(file); | |
262 | + const char *end = cp + checksum_len; | |
263 | + if (!(file->flags & FLAG_FILE_SENT)) | |
264 | + continue; | |
265 | + while (cp != end) | |
266 | + fprintf(out_fp, "%02x", CVAL(cp++, 0)); | |
267 | + fprintf(out_fp, " %10.0f %10ld %s\n", | |
268 | + (double)F_LENGTH(file), (long)file->modtime, | |
269 | + file->basename); | |
270 | + } | |
271 | + | |
272 | + fclose(out_fp); | |
273 | +} | |
274 | + | |
275 | int push_pathname(const char *dir, int len) | |
276 | { | |
277 | if (dir == pathname) | |
278 | @@ -973,34 +1187,24 @@ static struct file_struct *recv_file_ent | |
279 | return file; | |
280 | } | |
281 | ||
282 | -/** | |
283 | - * Create a file_struct for a named file by reading its stat() | |
284 | - * information and performing extensive checks against global | |
285 | - * options. | |
286 | - * | |
287 | - * @return the new file, or NULL if there was an error or this file | |
288 | - * should be excluded. | |
289 | +/* Create a file_struct for a named file by reading its stat() information | |
290 | + * and performing extensive checks against global options. | |
291 | * | |
292 | - * @todo There is a small optimization opportunity here to avoid | |
293 | - * stat()ing the file in some circumstances, which has a certain cost. | |
294 | - * We are called immediately after doing readdir(), and so we may | |
295 | - * already know the d_type of the file. We could for example avoid | |
296 | - * statting directories if we're not recursing, but this is not a very | |
297 | - * important case. Some systems may not have d_type. | |
298 | - **/ | |
299 | + * Returns a pointer to the new file struct, or NULL if there was an error | |
300 | + * or this file should be excluded. */ | |
301 | struct file_struct *make_file(const char *fname, struct file_list *flist, | |
302 | STRUCT_STAT *stp, int flags, int filter_level) | |
303 | { | |
304 | static char *lastdir; | |
305 | - static int lastdir_len = -1; | |
306 | + static int lastdir_len = -2; | |
307 | struct file_struct *file; | |
308 | - STRUCT_STAT st; | |
309 | char thisname[MAXPATHLEN]; | |
310 | char linkname[MAXPATHLEN]; | |
311 | int alloc_len, basename_len, linkname_len; | |
312 | int extra_len = file_extra_cnt * EXTRA_LEN; | |
313 | const char *basename; | |
314 | alloc_pool_t *pool; | |
315 | + STRUCT_STAT st; | |
316 | char *bp; | |
317 | ||
318 | if (strlcpy(thisname, fname, sizeof thisname) | |
319 | @@ -1115,9 +1319,16 @@ struct file_struct *make_file(const char | |
320 | memcpy(lastdir, thisname, len); | |
321 | lastdir[len] = '\0'; | |
322 | lastdir_len = len; | |
323 | + if (always_checksum && am_sender && flist) | |
324 | + read_checksums(lastdir); | |
325 | } | |
326 | - } else | |
327 | + } else { | |
328 | basename = thisname; | |
329 | + if (always_checksum && am_sender && flist && lastdir_len == -2) { | |
330 | + lastdir_len = -1; | |
331 | + read_checksums(NULL); | |
332 | + } | |
333 | + } | |
334 | basename_len = strlen(basename) + 1; /* count the '\0' */ | |
335 | ||
336 | #ifdef SUPPORT_LINKS | |
337 | @@ -1193,11 +1404,30 @@ struct file_struct *make_file(const char | |
338 | } | |
339 | #endif | |
340 | ||
341 | - if (always_checksum && am_sender && S_ISREG(st.st_mode)) | |
342 | - file_checksum(thisname, tmp_sum, st.st_size); | |
343 | - | |
344 | F_PATHNAME(file) = pathname; | |
345 | ||
346 | + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { | |
347 | + int j; | |
348 | + if (flist && (j = flist_find(checksum_flist, file)) >= 0) { | |
349 | + struct file_struct *fp = checksum_flist->sorted[j]; | |
350 | + if (fp->modtime == file->modtime && F_LENGTH(fp) == F_LENGTH(file)) { | |
351 | + memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); | |
352 | + fp->flags |= FLAG_FILE_SENT; | |
353 | + checksum_matches++; | |
354 | + } else { | |
355 | + clear_file(fp); | |
356 | + goto compute_checksum; | |
357 | + } | |
358 | + } else { | |
359 | + compute_checksum: | |
360 | + file_checksum(thisname, tmp_sum, st.st_size); | |
361 | + if (checksum_updating && flist) { | |
362 | + add_checksum(file->dirname, basename, basename_len, | |
363 | + st.st_size, st.st_mtime, tmp_sum, 0); | |
364 | + } | |
365 | + } | |
366 | + } | |
367 | + | |
368 | /* This code is only used by the receiver when it is building | |
369 | * a list of files for a delete pass. */ | |
370 | if (keep_dirlinks && linkname_len && flist) { | |
371 | @@ -1241,14 +1471,14 @@ void unmake_file(struct file_struct *fil | |
372 | ||
373 | static struct file_struct *send_file_name(int f, struct file_list *flist, | |
374 | char *fname, STRUCT_STAT *stp, | |
375 | - int flags, int filter_flags) | |
376 | + int flags, int filter_level) | |
377 | { | |
378 | struct file_struct *file; | |
379 | #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS | |
380 | statx sx; | |
381 | #endif | |
382 | ||
383 | - file = make_file(fname, flist, stp, flags, filter_flags); | |
384 | + file = make_file(fname, flist, stp, flags, filter_level); | |
385 | if (!file) | |
386 | return NULL; | |
387 | ||
388 | @@ -1442,7 +1672,7 @@ static void send_directory(int f, struct | |
389 | DIR *d; | |
390 | int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0; | |
391 | int start = flist->count; | |
392 | - int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS; | |
393 | + int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; | |
394 | ||
395 | assert(flist != NULL); | |
396 | ||
397 | @@ -1471,7 +1701,7 @@ static void send_directory(int f, struct | |
398 | continue; | |
399 | } | |
400 | ||
401 | - send_file_name(f, flist, fbuf, NULL, flags, filter_flags); | |
402 | + send_file_name(f, flist, fbuf, NULL, flags, filter_level); | |
403 | } | |
404 | ||
405 | fbuf[len] = '\0'; | |
406 | @@ -1483,6 +1713,9 @@ static void send_directory(int f, struct | |
407 | ||
408 | closedir(d); | |
409 | ||
410 | + if (checksum_updating && always_checksum && am_sender && f >= 0) | |
411 | + write_checksums(fbuf); | |
412 | + | |
413 | if (f >= 0 && recurse && !divert_dirs) { | |
414 | int i, end = flist->count - 1; | |
415 | /* send_if_directory() bumps flist->count, so use "end". */ | |
416 | @@ -2206,7 +2439,7 @@ void flist_free(struct file_list *flist) | |
417 | ||
418 | if (!flist->prev || !flist_cnt) | |
419 | pool_destroy(flist->file_pool); | |
420 | - else | |
421 | + else if (flist->pool_boundary) | |
422 | pool_free_old(flist->file_pool, flist->pool_boundary); | |
423 | ||
424 | if (flist->sorted && flist->sorted != flist->files) | |
425 | @@ -2225,6 +2458,7 @@ static void clean_flist(struct file_list | |
426 | if (!flist) | |
427 | return; | |
428 | if (flist->count == 0) { | |
429 | + flist->low = 0; | |
430 | flist->high = -1; | |
431 | return; | |
432 | } | |
433 | --- old/loadparm.c | |
434 | +++ new/loadparm.c | |
435 | @@ -149,6 +149,7 @@ typedef struct | |
436 | int syslog_facility; | |
437 | int timeout; | |
438 | ||
439 | + BOOL checksum_updating; | |
440 | BOOL fake_super; | |
441 | BOOL ignore_errors; | |
442 | BOOL ignore_nonreadable; | |
443 | @@ -197,6 +198,7 @@ static service sDefault = | |
444 | /* syslog_facility; */ LOG_DAEMON, | |
445 | /* timeout; */ 0, | |
446 | ||
447 | + /* checksum_updating; */ False, | |
448 | /* fake_super; */ False, | |
449 | /* ignore_errors; */ False, | |
450 | /* ignore_nonreadable; */ False, | |
451 | @@ -313,6 +315,7 @@ static struct parm_struct parm_table[] = | |
452 | {"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0}, | |
453 | {"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0}, | |
454 | {"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0}, | |
455 | + {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0}, | |
456 | {"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0}, | |
457 | {"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0}, | |
458 | {"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0}, | |
459 | @@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super) | |
460 | FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) | |
461 | FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) | |
462 | FN_LOCAL_BOOL(lp_list, list) | |
463 | +FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating) | |
464 | FN_LOCAL_BOOL(lp_read_only, read_only) | |
465 | FN_LOCAL_BOOL(lp_strict_modes, strict_modes) | |
466 | FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging) | |
467 | --- old/options.c | |
468 | +++ new/options.c | |
469 | @@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0; | |
470 | int ignore_existing = 0; | |
471 | int ignore_non_existing = 0; | |
472 | int need_messages_from_generator = 0; | |
473 | +int checksum_updating = 0; | |
474 | int max_delete = -1; | |
475 | OFF_T max_size = 0; | |
476 | OFF_T min_size = 0; | |
477 | @@ -302,6 +303,7 @@ void usage(enum logcode F) | |
478 | rprintf(F," -q, --quiet suppress non-error messages\n"); | |
479 | rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); | |
480 | rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); | |
481 | + rprintf(F," --checksum-updating sender updates .md[45]sums files\n"); | |
482 | rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); | |
483 | rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); | |
484 | rprintf(F," -r, --recursive recurse into directories\n"); | |
485 | @@ -542,6 +544,7 @@ static struct poptOption long_options[] | |
486 | {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, | |
487 | {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
488 | {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
489 | + {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 }, | |
490 | {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, | |
491 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, | |
492 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, | |
493 | @@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg | |
494 | args[ac++] = basis_dir[i]; | |
495 | } | |
496 | } | |
497 | - } | |
498 | + } else if (checksum_updating) | |
499 | + args[ac++] = "--checksum-updating"; | |
500 | + | |
501 | ||
502 | if (append_mode) | |
503 | args[ac++] = "--append"; | |
504 | --- old/rsync.h | |
505 | +++ new/rsync.h | |
506 | @@ -1070,6 +1070,12 @@ isDigit(const char *ptr) | |
507 | } | |
508 | ||
509 | static inline int | |
510 | +isAlpha(const char *ptr) | |
511 | +{ | |
512 | + return isalpha(*(unsigned char *)ptr); | |
513 | +} | |
514 | + | |
515 | +static inline int | |
516 | isPrint(const char *ptr) | |
517 | { | |
518 | return isprint(*(unsigned char *)ptr); | |
519 | --- old/rsync.yo | |
520 | +++ new/rsync.yo | |
521 | @@ -307,6 +307,7 @@ to the detailed description below for a | |
522 | -q, --quiet suppress non-error messages | |
523 | --no-motd suppress daemon-mode MOTD (see caveat) | |
524 | -c, --checksum skip based on checksum, not mod-time & size | |
525 | + --checksum-updating sender updates .md[45]sums files | |
526 | -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) | |
527 | --no-OPTION turn off an implied OPTION (e.g. --no-D) | |
528 | -r, --recursive recurse into directories | |
529 | @@ -502,9 +503,9 @@ uses a "quick check" that (by default) c | |
530 | of last modification match between the sender and receiver. This option | |
531 | changes this to compare a 128-bit MD4 checksum for each file that has a | |
532 | matching size. Generating the checksums means that both sides will expend | |
533 | -a lot of disk I/O reading all the data in the files in the transfer (and | |
534 | -this is prior to any reading that will be done to transfer changed files), | |
535 | -so this can slow things down significantly. | |
536 | +a lot of disk I/O reading the data in all the files in the transfer, so | |
537 | +this can slow things down significantly (and this is prior to any reading | |
538 | +that will be done to transfer the files that have changed). | |
539 | ||
540 | The sending side generates its checksums while it is doing the file-system | |
541 | scan that builds the list of the available files. The receiver generates | |
542 | @@ -512,12 +513,43 @@ its checksums when it is scanning for ch | |
543 | file that has the same size as the corresponding sender's file: files with | |
544 | either a changed size or a changed checksum are selected for transfer. | |
545 | ||
546 | +Starting with version 3.0.0, the sending side will look for a checksum | |
547 | +summary file and use a pre-generated checksum that it reads out of the file | |
548 | +(as long as it matches the file's size and modified time). This allows a | |
549 | +server to support the --checksum option to clients without having to | |
550 | +recompute the checksums for each client. See the bf(--checksum-updating) | |
551 | +option for a way to have rsync create/update the checksum files. | |
552 | + | |
553 | Note that rsync always verifies that each em(transferred) file was | |
554 | correctly reconstructed on the receiving side by checking a whole-file | |
555 | checksum that is generated when as the file is transferred, but that | |
556 | automatic after-the-transfer verification has nothing to do with this | |
557 | option's before-the-transfer "Does this file need to be updated?" check. | |
558 | ||
559 | +dit(bf(--checksum-updating)) This option tells the sending side to create | |
560 | +and/or update per-directory checksum files that are used by the | |
561 | +bf(--checksum) option. The file that is updated is either .md5sums (for | |
562 | +protocols >= 30) or .md4sums (for older protocols). If pre-transfer | |
563 | +checksums are not being computed, this option has no effect. | |
564 | + | |
565 | +The checksum files stores the computed checksum, last-known size, | |
566 | +modification time, and name for each file in the current directory. If a | |
567 | +later transfer finds that a file matches its prior size and modification | |
568 | +time, the checksum is assumed to still be correct. Otherwise it is | |
569 | +recomputed and udpated in the file. | |
570 | + | |
571 | +To avoid transferring the system's checksum files, you can use an exclude | |
572 | +(e.g. bf(--exclude=.md[45]sums)). To make this easier to type, you can use | |
573 | +a popt alias. For instance, adding the following line in your ~/.popt file | |
574 | +defines a bf(-cc) option that enables checksum updating and excludes the | |
575 | +checksum files: | |
576 | + | |
577 | +verb( rsync alias --cc --checksum-updating --exclude='.md[45]sums') | |
578 | + | |
579 | +An rsync daemon does not allow the client to control this setting, so see | |
580 | +the "checksum updating" daemon config option for information on how to make | |
581 | +a daemon maintain these checksum files. | |
582 | + | |
583 | dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick | |
584 | way of saying you want recursion and want to preserve almost | |
585 | everything (with -H being a notable omission). | |
586 | --- old/rsyncd.conf.yo | |
587 | +++ new/rsyncd.conf.yo | |
588 | @@ -198,6 +198,21 @@ locking on this file to ensure that the | |
589 | exceeded for the modules sharing the lock file. | |
590 | The default is tt(/var/run/rsyncd.lock). | |
591 | ||
592 | +dit(bf(checksum updating)) This option tells rsync to update/create the | |
593 | +checksum information in the per-directory checksum files when users copy | |
594 | +files using the bf(--checksum) option. Any file that has changed since it | |
595 | +was last checksummed (or is not mentioned) has its data updated in the | |
596 | +.md4sums or .md5sums file (the file used depends on what protocol version | |
597 | +is used for the transfer). | |
598 | + | |
599 | +Note that this updating will occur even if the module is listed as being | |
600 | +read-only. If you want to hide these files (and you will almost always | |
601 | +want to do), add ".md[45]sums" to the module's exclude setting. | |
602 | + | |
603 | +Note also that the client's command-line option, bf(--checksum-updating), | |
604 | +has no effect on a daemon. A daemon will only update/create checksum files | |
605 | +if this config option is true. | |
606 | + | |
607 | dit(bf(read only)) The "read only" option determines whether clients | |
608 | will be able to upload files or not. If "read only" is true then any | |
609 | attempted uploads will fail. If "read only" is false then uploads will |