Commit | Line | Data |
---|---|---|
ae10e51e | 1 | This adds a sender optimization feature that allows a cache of checksums |
56522462 WD |
2 | to be used when the client specifies the --checksum option, and creates |
3 | and/or updates the .rsyncsums files when --checksum-updating is | |
4 | specified. | |
ae10e51e WD |
5 | |
6 | To use this patch, run these commands for a successful build: | |
7 | ||
8 | patch -p1 <patches/checksum-updating.diff | |
9 | ./configure (optional if already run) | |
10 | make | |
11 | ||
ae10e51e WD |
12 | --- old/clientserver.c |
13 | +++ new/clientserver.c | |
a3ba6627 | 14 | @@ -37,6 +37,7 @@ extern int sanitize_paths; |
ae10e51e WD |
15 | extern int filesfrom_fd; |
16 | extern int remote_protocol; | |
17 | extern int protocol_version; | |
18 | +extern int checksum_updating; | |
19 | extern int io_timeout; | |
20 | extern int no_detach; | |
21 | extern int default_af_hint; | |
a3ba6627 | 22 | @@ -641,6 +642,8 @@ static int rsync_module(int f_in, int f_ |
ae10e51e WD |
23 | else if (am_root < 0) /* Treat --fake-super from client as --super. */ |
24 | am_root = 2; | |
25 | ||
26 | + checksum_updating = lp_checksum_updating(i); | |
27 | + | |
28 | if (filesfrom_fd == 0) | |
29 | filesfrom_fd = f_in; | |
30 | ||
31 | --- old/flist.c | |
32 | +++ new/flist.c | |
a3ba6627 | 33 | @@ -25,6 +25,7 @@ |
ae10e51e WD |
34 | #include "io.h" |
35 | ||
36 | extern int verbose; | |
37 | +extern int dry_run; | |
38 | extern int list_only; | |
39 | extern int am_root; | |
40 | extern int am_server; | |
a3ba6627 | 41 | @@ -57,6 +58,7 @@ extern int implied_dirs; |
ae10e51e WD |
42 | extern int file_extra_cnt; |
43 | extern int ignore_perishable; | |
44 | extern int non_perishable_cnt; | |
45 | +extern int checksum_updating; | |
46 | extern int prune_empty_dirs; | |
47 | extern int copy_links; | |
48 | extern int copy_unsafe_links; | |
a3ba6627 | 49 | @@ -79,6 +81,9 @@ extern iconv_t ic_send, ic_recv; |
56522462 WD |
50 | |
51 | #define PTR_SIZE (sizeof (struct file_struct *)) | |
52 | ||
edf38a9d WD |
53 | +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */ |
54 | +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */ | |
56522462 WD |
55 | + |
56 | int io_error; | |
57 | int checksum_len; | |
58 | dev_t filesystem_dev; /* used to implement -x */ | |
a3ba6627 | 59 | @@ -101,6 +106,10 @@ static char tmp_sum[MAX_DIGEST_LEN]; |
ae10e51e WD |
60 | static char empty_sum[MAX_DIGEST_LEN]; |
61 | static int flist_count_offset; /* for --delete --progress */ | |
62 | static int dir_count = 0; | |
63 | +static struct file_list *checksum_flist = NULL; | |
64 | +static int checksum_matches = 0; | |
071bf6df | 65 | +static int checksum_updates = 0; |
7200c744 | 66 | +static int regular_skipped = 0; |
ae10e51e WD |
67 | |
68 | static void clean_flist(struct file_list *flist, int strip_root); | |
69 | static void output_flist(struct file_list *flist); | |
a3ba6627 | 70 | @@ -317,6 +326,304 @@ static void flist_done_allocating(struct |
ae10e51e WD |
71 | flist->pool_boundary = ptr; |
72 | } | |
73 | ||
74 | +/* The len count is the length of the basename + 1 for the null. */ | |
75 | +static void add_checksum(const char *dirname, const char *basename, int len, | |
7200c744 WD |
76 | + OFF_T file_length, time_t mtime, time_t ctime, |
77 | + const char *sum, const char *alt_sum, int flags) | |
ae10e51e WD |
78 | +{ |
79 | + struct file_struct *file; | |
80 | + int alloc_len, extra_len; | |
81 | + char *bp; | |
82 | + | |
56522462 | 83 | + if (len == 10+1 && *basename == '.' && strcmp(basename, ".rsyncsums") == 0) |
ae10e51e WD |
84 | + return; |
85 | + | |
86 | + if (len < 0) | |
87 | + len = strlen(basename) + 1; | |
88 | + | |
7200c744 WD |
89 | + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) |
90 | + + SUM_EXTRA_CNT + TIME_EXTRA_CNT) | |
91 | + * EXTRA_LEN; | |
ae10e51e WD |
92 | +#if EXTRA_ROUNDING > 0 |
93 | + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) | |
94 | + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; | |
95 | +#endif | |
56522462 | 96 | + alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1; |
ae10e51e WD |
97 | + bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); |
98 | + | |
99 | + memset(bp, 0, extra_len + FILE_STRUCT_LEN); | |
100 | + bp += extra_len; | |
101 | + file = (struct file_struct *)bp; | |
102 | + bp += FILE_STRUCT_LEN; | |
103 | + | |
104 | + memcpy(bp, basename, len); | |
56522462 WD |
105 | + if (alt_sum) |
106 | + strlcpy(bp+len, alt_sum, checksum_len*2 + 1); | |
107 | + else { | |
108 | + memset(bp+len, '=', checksum_len*2); | |
109 | + bp[len+checksum_len*2] = '\0'; | |
110 | + } | |
ae10e51e WD |
111 | + |
112 | + file->flags = flags; | |
113 | + file->mode = S_IFREG; | |
114 | + file->modtime = mtime; | |
115 | + file->len32 = (uint32)file_length; | |
116 | + if (file_length > 0xFFFFFFFFu) { | |
117 | + file->flags |= FLAG_LENGTH64; | |
118 | + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); | |
119 | + } | |
120 | + file->dirname = dirname; | |
a3ba6627 | 121 | + bp = F_SUM(file); |
ae10e51e | 122 | + memcpy(bp, sum, checksum_len); |
edf38a9d WD |
123 | +#if SIZEOF_TIME_T == 4 |
124 | + OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num = ctime; | |
125 | +#else | |
7200c744 | 126 | + memcpy(bp - SIZEOF_TIME_T, &ctime, SIZEOF_TIME_T); |
edf38a9d | 127 | +#endif |
ae10e51e WD |
128 | + |
129 | + flist_expand(checksum_flist, 1); | |
130 | + checksum_flist->files[checksum_flist->count++] = file; | |
131 | + | |
132 | + checksum_flist->sorted = checksum_flist->files; | |
133 | +} | |
134 | + | |
071bf6df WD |
135 | +static void write_checksums(const char *next_dirname, int whole_dir) |
136 | +{ | |
137 | + static const char *dirname_save; | |
138 | + char fbuf[MAXPATHLEN]; | |
139 | + const char *dirname; | |
140 | + int count, new_entries, counts_match, no_skipped; | |
141 | + time_t latest_time = 0; | |
142 | + FILE *out_fp; | |
143 | + int i; | |
144 | + | |
145 | + dirname = dirname_save; | |
146 | + dirname_save = next_dirname; | |
147 | + | |
148 | + if (!dirname) | |
149 | + return; | |
150 | + | |
151 | + count = checksum_flist->count; | |
152 | + new_entries = checksum_updates != 0; | |
153 | + counts_match = count == checksum_matches; | |
154 | + no_skipped = whole_dir && regular_skipped == 0; | |
155 | + | |
156 | + clean_flist(checksum_flist, 0); | |
157 | + | |
158 | + checksum_flist->count = 0; | |
159 | + checksum_matches = 0; | |
160 | + checksum_updates = 0; | |
161 | + regular_skipped = 0; | |
162 | + | |
163 | + if (dry_run) | |
164 | + return; | |
165 | + | |
166 | + if (*dirname) { | |
167 | + if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf) | |
168 | + return; | |
169 | + } else | |
170 | + strlcpy(fbuf, ".rsyncsums", sizeof fbuf); | |
171 | + | |
172 | + if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) { | |
173 | + unlink(fbuf); | |
174 | + return; | |
175 | + } | |
176 | + | |
177 | + if (!new_entries && (counts_match || !whole_dir)) | |
178 | + return; | |
179 | + | |
180 | + if (!(out_fp = fopen(fbuf, "w"))) | |
181 | + return; | |
182 | + | |
183 | + new_entries = 0; | |
184 | + for (i = checksum_flist->low; i <= checksum_flist->high; i++) { | |
185 | + struct file_struct *file = checksum_flist->sorted[i]; | |
186 | + const char *cp = F_SUM(file); | |
187 | + const char *end = cp + checksum_len; | |
188 | + const char *alt_sum = file->basename + strlen(file->basename) + 1; | |
189 | + time_t ctime; | |
190 | + if (whole_dir && !(file->flags & FLAG_SUM_KEEP)) | |
191 | + continue; | |
192 | +#if SIZEOF_TIME_T == 4 | |
193 | + ctime = OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num; | |
194 | +#else | |
195 | + memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T); | |
196 | +#endif | |
197 | + if (protocol_version >= 30) | |
198 | + fprintf(out_fp, "%s ", alt_sum); | |
199 | + if (file->flags & FLAG_SUM_MISSING) { | |
200 | + new_entries++; | |
201 | + do { | |
202 | + fprintf(out_fp, "=="); | |
203 | + } while (++cp != end); | |
204 | + } else { | |
205 | + do { | |
206 | + fprintf(out_fp, "%02x", CVAL(cp, 0)); | |
207 | + } while (++cp != end); | |
208 | + } | |
209 | + if (protocol_version < 30) | |
210 | + fprintf(out_fp, " %s", alt_sum); | |
211 | + if (*alt_sum == '=') | |
212 | + new_entries++; | |
213 | + fprintf(out_fp, " %10.0f %10.0f %10.0f %s\n", | |
214 | + (double)F_LENGTH(file), (double)file->modtime, | |
215 | + (double)ctime, file->basename); | |
216 | + if (file->modtime > ctime) | |
217 | + ctime = file->modtime; | |
218 | + if (ctime > latest_time) | |
219 | + latest_time = ctime; | |
220 | + } | |
221 | + | |
222 | + fclose(out_fp); | |
223 | + | |
224 | + if (whole_dir && new_entries == 0) | |
225 | + set_modtime(fbuf, latest_time, latest_time); | |
226 | + else | |
227 | + set_modtime(fbuf, latest_time-1, latest_time-1); | |
228 | +} | |
229 | + | |
ae10e51e WD |
230 | +/* The direname value must remain unchanged during the lifespan of the |
231 | + * created checksum_flist object because we use it directly. */ | |
232 | +static void read_checksums(const char *dirname) | |
233 | +{ | |
234 | + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; | |
56522462 | 235 | + const char *alt_sum = NULL; |
ae10e51e | 236 | + OFF_T file_length; |
7200c744 | 237 | + time_t mtime, ctime; |
ae10e51e WD |
238 | + int len, dlen, i, flags; |
239 | + char *cp; | |
240 | + FILE *fp; | |
241 | + | |
071bf6df WD |
242 | + write_checksums(dirname, 0); |
243 | + | |
ae10e51e WD |
244 | + if (checksum_flist) { |
245 | + /* Reset the pool memory and empty the file-list array. */ | |
246 | + pool_free_old(checksum_flist->file_pool, | |
247 | + pool_boundary(checksum_flist->file_pool, 0)); | |
248 | + checksum_flist->count = 0; | |
249 | + } else | |
250 | + checksum_flist = flist_new(FLIST_TEMP, "read_checksums"); | |
251 | + | |
252 | + checksum_flist->low = 0; | |
253 | + checksum_flist->high = -1; | |
254 | + checksum_matches = 0; | |
071bf6df | 255 | + checksum_updates = 0; |
7200c744 | 256 | + regular_skipped = 0; |
ae10e51e | 257 | + |
071bf6df WD |
258 | + if (!dirname) |
259 | + return; | |
260 | + | |
261 | + dlen = strlcpy(fbuf, dirname, sizeof fbuf); | |
262 | + if (dlen >= (int)sizeof fbuf) | |
263 | + return; | |
264 | + if (dlen) | |
ae10e51e | 265 | + fbuf[dlen++] = '/'; |
071bf6df WD |
266 | + else |
267 | + dirname = NULL; | |
56522462 | 268 | + strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen); |
ae10e51e WD |
269 | + if (!(fp = fopen(fbuf, "r"))) |
270 | + return; | |
271 | + | |
272 | + while (fgets(line, sizeof line, fp)) { | |
56522462 WD |
273 | + cp = line; |
274 | + if (protocol_version >= 30) { | |
275 | + alt_sum = cp; | |
276 | + if (*cp == '=') | |
277 | + while (*++cp == '=') {} | |
278 | + else | |
279 | + while (isXDigit(cp)) cp++; | |
280 | + if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ') | |
281 | + break; | |
282 | + while (*++cp == ' ') {} | |
283 | + } | |
284 | + | |
285 | + if (*cp == '=') { | |
286 | + for (i = 0; i < checksum_len*2; i++, cp++) { | |
287 | + if (*cp != '=') { | |
ae10e51e WD |
288 | + cp = ""; |
289 | + break; | |
290 | + } | |
ae10e51e | 291 | + } |
56522462 WD |
292 | + memset(sum, 0, checksum_len); |
293 | + flags = FLAG_SUM_MISSING; | |
294 | + } else { | |
295 | + for (i = 0; i < checksum_len*2; i++, cp++) { | |
296 | + int x; | |
297 | + if (isXDigit(cp)) { | |
298 | + if (isDigit(cp)) | |
299 | + x = *cp - '0'; | |
300 | + else | |
301 | + x = (*cp & 0xF) + 9; | |
302 | + } else { | |
303 | + cp = ""; | |
304 | + break; | |
305 | + } | |
306 | + if (i & 1) | |
307 | + sum[i/2] |= x; | |
308 | + else | |
309 | + sum[i/2] = x << 4; | |
310 | + } | |
311 | + flags = 0; | |
ae10e51e | 312 | + } |
ae10e51e | 313 | + if (*cp != ' ') |
56522462 | 314 | + break; |
ae10e51e WD |
315 | + while (*++cp == ' ') {} |
316 | + | |
56522462 WD |
317 | + if (protocol_version < 30) { |
318 | + alt_sum = cp; | |
319 | + if (*cp == '=') | |
320 | + while (*++cp == '=') {} | |
321 | + else | |
322 | + while (isXDigit(cp)) cp++; | |
323 | + if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ') | |
324 | + break; | |
325 | + while (*++cp == ' ') {} | |
326 | + } | |
327 | + | |
ae10e51e WD |
328 | + file_length = 0; |
329 | + while (isDigit(cp)) | |
330 | + file_length = file_length * 10 + *cp++ - '0'; | |
ae10e51e | 331 | + if (*cp != ' ') |
56522462 | 332 | + break; |
ae10e51e WD |
333 | + while (*++cp == ' ') {} |
334 | + | |
335 | + mtime = 0; | |
336 | + while (isDigit(cp)) | |
337 | + mtime = mtime * 10 + *cp++ - '0'; | |
ae10e51e | 338 | + if (*cp != ' ') |
56522462 | 339 | + break; |
ae10e51e WD |
340 | + while (*++cp == ' ') {} |
341 | + | |
7200c744 WD |
342 | + ctime = 0; |
343 | + while (isDigit(cp)) | |
344 | + ctime = ctime * 10 + *cp++ - '0'; | |
345 | + if (*cp != ' ') | |
346 | + break; | |
347 | + while (*++cp == ' ') {} | |
348 | + | |
ae10e51e WD |
349 | + len = strlen(cp); |
350 | + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r')) | |
351 | + len--; | |
352 | + if (!len) | |
56522462 | 353 | + break; |
ae10e51e WD |
354 | + cp[len++] = '\0'; /* len now counts the null */ |
355 | + if (strchr(cp, '/') || len > MAXPATHLEN) | |
56522462 | 356 | + break; |
ae10e51e WD |
357 | + |
358 | + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); | |
359 | + if (is_excluded(fbuf, 0, ALL_FILTERS)) { | |
edf38a9d | 360 | + flags |= FLAG_SUM_KEEP; |
ae10e51e | 361 | + checksum_matches++; |
56522462 | 362 | + } |
ae10e51e | 363 | + |
7200c744 | 364 | + add_checksum(dirname, cp, len, file_length, mtime, ctime, |
56522462 | 365 | + sum, alt_sum, flags); |
ae10e51e WD |
366 | + } |
367 | + fclose(fp); | |
368 | + | |
369 | + clean_flist(checksum_flist, 0); | |
370 | +} | |
ae10e51e WD |
371 | + |
372 | int push_pathname(const char *dir, int len) | |
373 | { | |
374 | if (dir == pathname) | |
99650e0d | 375 | @@ -980,34 +1287,24 @@ static struct file_struct *recv_file_ent |
ae10e51e WD |
376 | return file; |
377 | } | |
378 | ||
379 | -/** | |
380 | - * Create a file_struct for a named file by reading its stat() | |
381 | - * information and performing extensive checks against global | |
382 | - * options. | |
383 | - * | |
384 | - * @return the new file, or NULL if there was an error or this file | |
385 | - * should be excluded. | |
386 | +/* Create a file_struct for a named file by reading its stat() information | |
387 | + * and performing extensive checks against global options. | |
388 | * | |
389 | - * @todo There is a small optimization opportunity here to avoid | |
390 | - * stat()ing the file in some circumstances, which has a certain cost. | |
391 | - * We are called immediately after doing readdir(), and so we may | |
392 | - * already know the d_type of the file. We could for example avoid | |
393 | - * statting directories if we're not recursing, but this is not a very | |
394 | - * important case. Some systems may not have d_type. | |
395 | - **/ | |
396 | + * Returns a pointer to the new file struct, or NULL if there was an error | |
397 | + * or this file should be excluded. */ | |
398 | struct file_struct *make_file(const char *fname, struct file_list *flist, | |
399 | STRUCT_STAT *stp, int flags, int filter_level) | |
400 | { | |
401 | static char *lastdir; | |
402 | - static int lastdir_len = -1; | |
403 | + static int lastdir_len = -2; | |
404 | struct file_struct *file; | |
405 | - STRUCT_STAT st; | |
406 | char thisname[MAXPATHLEN]; | |
407 | char linkname[MAXPATHLEN]; | |
408 | int alloc_len, basename_len, linkname_len; | |
409 | int extra_len = file_extra_cnt * EXTRA_LEN; | |
410 | const char *basename; | |
411 | alloc_pool_t *pool; | |
412 | + STRUCT_STAT st; | |
413 | char *bp; | |
414 | ||
9c85142a | 415 | if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) { |
99650e0d | 416 | @@ -1086,6 +1383,8 @@ struct file_struct *make_file(const char |
7200c744 WD |
417 | if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { |
418 | if (ignore_perishable) | |
419 | non_perishable_cnt++; | |
420 | + if (S_ISREG(st.st_mode)) | |
421 | + regular_skipped++; | |
422 | return NULL; | |
423 | } | |
424 | ||
99650e0d | 425 | @@ -1124,9 +1423,16 @@ struct file_struct *make_file(const char |
ae10e51e WD |
426 | memcpy(lastdir, thisname, len); |
427 | lastdir[len] = '\0'; | |
428 | lastdir_len = len; | |
429 | + if (always_checksum && am_sender && flist) | |
430 | + read_checksums(lastdir); | |
431 | } | |
432 | - } else | |
433 | + } else { | |
434 | basename = thisname; | |
435 | + if (always_checksum && am_sender && flist && lastdir_len == -2) { | |
436 | + lastdir_len = -1; | |
071bf6df | 437 | + read_checksums(""); |
ae10e51e WD |
438 | + } |
439 | + } | |
440 | basename_len = strlen(basename) + 1; /* count the '\0' */ | |
441 | ||
442 | #ifdef SUPPORT_LINKS | |
99650e0d | 443 | @@ -1202,11 +1508,44 @@ struct file_struct *make_file(const char |
ae10e51e WD |
444 | } |
445 | #endif | |
446 | ||
447 | - if (always_checksum && am_sender && S_ISREG(st.st_mode)) | |
448 | - file_checksum(thisname, tmp_sum, st.st_size); | |
449 | - | |
450 | F_PATHNAME(file) = pathname; | |
451 | ||
452 | + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { | |
453 | + int j; | |
454 | + if (flist && (j = flist_find(checksum_flist, file)) >= 0) { | |
455 | + struct file_struct *fp = checksum_flist->sorted[j]; | |
7200c744 | 456 | + time_t ctime; |
edf38a9d WD |
457 | +#if SIZEOF_TIME_T == 4 |
458 | + ctime = OPT_EXTRA(fp, LEN64_BUMP(fp) + SUM_EXTRA_CNT)->num; | |
459 | +#else | |
7200c744 | 460 | + memcpy(&ctime, F_SUM(fp) - SIZEOF_TIME_T, SIZEOF_TIME_T); |
edf38a9d | 461 | +#endif |
7200c744 WD |
462 | + if (F_LENGTH(fp) == st.st_size |
463 | + && fp->modtime == st.st_mtime && ctime == st.st_ctime) { | |
56522462 WD |
464 | + if (fp->flags & FLAG_SUM_MISSING) { |
465 | + fp->flags &= ~FLAG_SUM_MISSING; | |
466 | + file_checksum(thisname, tmp_sum, st.st_size); | |
a3ba6627 | 467 | + memcpy(F_SUM(fp), tmp_sum, MAX_DIGEST_LEN); |
56522462 WD |
468 | + } else { |
469 | + checksum_matches++; | |
470 | + memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); | |
471 | + } | |
edf38a9d | 472 | + fp->flags |= FLAG_SUM_KEEP; |
ae10e51e WD |
473 | + } else { |
474 | + clear_file(fp); | |
edf38a9d | 475 | + goto compute_new_checksum; |
ae10e51e WD |
476 | + } |
477 | + } else { | |
edf38a9d | 478 | + compute_new_checksum: |
ae10e51e WD |
479 | + file_checksum(thisname, tmp_sum, st.st_size); |
480 | + if (checksum_updating && flist) { | |
481 | + add_checksum(file->dirname, basename, basename_len, | |
7200c744 | 482 | + st.st_size, st.st_mtime, st.st_ctime, |
edf38a9d | 483 | + tmp_sum, NULL, FLAG_SUM_KEEP); |
ae10e51e WD |
484 | + } |
485 | + } | |
486 | + } | |
487 | + | |
488 | /* This code is only used by the receiver when it is building | |
489 | * a list of files for a delete pass. */ | |
490 | if (keep_dirlinks && linkname_len && flist) { | |
99650e0d | 491 | @@ -1499,6 +1838,9 @@ static void send_directory(int f, struct |
ae10e51e WD |
492 | |
493 | closedir(d); | |
494 | ||
495 | + if (checksum_updating && always_checksum && am_sender && f >= 0) | |
071bf6df | 496 | + write_checksums(NULL, 1); |
ae10e51e WD |
497 | + |
498 | if (f >= 0 && recurse && !divert_dirs) { | |
9c85142a WD |
499 | int i, end = flist->used - 1; |
500 | /* send_if_directory() bumps flist->used, so use "end". */ | |
99650e0d | 501 | @@ -1942,7 +2284,11 @@ struct file_list *send_file_list(int f, |
071bf6df WD |
502 | * file-list to check if this is a 1-file xfer. */ |
503 | send_extra_file_list(f, 1); | |
504 | } | |
505 | - } | |
506 | + } else | |
507 | + flist_eof = 1; | |
508 | + | |
509 | + if (checksum_updating && always_checksum && flist_eof) | |
510 | + read_checksums(NULL); /* writes any last updates */ | |
511 | ||
512 | return flist; | |
513 | } | |
99650e0d | 514 | @@ -2235,7 +2581,7 @@ void flist_free(struct file_list *flist) |
ae10e51e WD |
515 | |
516 | if (!flist->prev || !flist_cnt) | |
517 | pool_destroy(flist->file_pool); | |
518 | - else | |
519 | + else if (flist->pool_boundary) | |
520 | pool_free_old(flist->file_pool, flist->pool_boundary); | |
521 | ||
522 | if (flist->sorted && flist->sorted != flist->files) | |
ae10e51e WD |
523 | --- old/loadparm.c |
524 | +++ new/loadparm.c | |
99650e0d | 525 | @@ -152,6 +152,7 @@ typedef struct |
ae10e51e WD |
526 | int syslog_facility; |
527 | int timeout; | |
528 | ||
529 | + BOOL checksum_updating; | |
530 | BOOL fake_super; | |
531 | BOOL ignore_errors; | |
532 | BOOL ignore_nonreadable; | |
99650e0d | 533 | @@ -200,6 +201,7 @@ static service sDefault = |
ae10e51e WD |
534 | /* syslog_facility; */ LOG_DAEMON, |
535 | /* timeout; */ 0, | |
536 | ||
537 | + /* checksum_updating; */ False, | |
538 | /* fake_super; */ False, | |
539 | /* ignore_errors; */ False, | |
540 | /* ignore_nonreadable; */ False, | |
99650e0d | 541 | @@ -316,6 +318,7 @@ static struct parm_struct parm_table[] = |
ae10e51e WD |
542 | {"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0}, |
543 | {"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0}, | |
544 | {"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0}, | |
545 | + {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0}, | |
546 | {"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0}, | |
547 | {"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0}, | |
548 | {"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0}, | |
99650e0d | 549 | @@ -421,6 +424,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super) |
ae10e51e WD |
550 | FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) |
551 | FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) | |
552 | FN_LOCAL_BOOL(lp_list, list) | |
553 | +FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating) | |
554 | FN_LOCAL_BOOL(lp_read_only, read_only) | |
555 | FN_LOCAL_BOOL(lp_strict_modes, strict_modes) | |
556 | FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging) | |
557 | --- old/options.c | |
558 | +++ new/options.c | |
a3ba6627 | 559 | @@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0; |
ae10e51e WD |
560 | int ignore_existing = 0; |
561 | int ignore_non_existing = 0; | |
562 | int need_messages_from_generator = 0; | |
563 | +int checksum_updating = 0; | |
99650e0d | 564 | int max_delete = INT_MIN; |
ae10e51e WD |
565 | OFF_T max_size = 0; |
566 | OFF_T min_size = 0; | |
99650e0d | 567 | @@ -303,6 +304,7 @@ void usage(enum logcode F) |
ae10e51e WD |
568 | rprintf(F," -q, --quiet suppress non-error messages\n"); |
569 | rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); | |
570 | rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); | |
56522462 | 571 | + rprintf(F," --checksum-updating sender updates .rsyncsums files\n"); |
ae10e51e WD |
572 | rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); |
573 | rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); | |
574 | rprintf(F," -r, --recursive recurse into directories\n"); | |
99650e0d | 575 | @@ -544,6 +546,7 @@ static struct poptOption long_options[] |
ae10e51e WD |
576 | {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, |
577 | {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
578 | {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
579 | + {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 }, | |
580 | {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, | |
581 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, | |
582 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, | |
99650e0d | 583 | @@ -1913,7 +1916,9 @@ void server_options(char **args,int *arg |
ae10e51e WD |
584 | args[ac++] = basis_dir[i]; |
585 | } | |
586 | } | |
587 | - } | |
588 | + } else if (checksum_updating) | |
589 | + args[ac++] = "--checksum-updating"; | |
590 | + | |
591 | ||
592 | if (append_mode) | |
593 | args[ac++] = "--append"; | |
594 | --- old/rsync.h | |
595 | +++ new/rsync.h | |
99650e0d | 596 | @@ -592,6 +592,7 @@ extern int xattrs_ndx; |
a3ba6627 WD |
597 | #define DEV_EXTRA_CNT 2 |
598 | #define DIRNODE_EXTRA_CNT 3 | |
7200c744 WD |
599 | #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN) |
600 | +#define TIME_EXTRA_CNT ((SIZEOF_TIME_T + EXTRA_LEN - 1) / EXTRA_LEN) | |
601 | ||
602 | #define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx)) | |
603 | #define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump)) | |
99650e0d | 604 | @@ -1077,6 +1078,12 @@ isDigit(const char *ptr) |
ae10e51e WD |
605 | } |
606 | ||
607 | static inline int | |
56522462 | 608 | +isXDigit(const char *ptr) |
ae10e51e | 609 | +{ |
56522462 | 610 | + return isxdigit(*(unsigned char *)ptr); |
ae10e51e WD |
611 | +} |
612 | + | |
613 | +static inline int | |
614 | isPrint(const char *ptr) | |
615 | { | |
616 | return isprint(*(unsigned char *)ptr); | |
617 | --- old/rsync.yo | |
618 | +++ new/rsync.yo | |
99650e0d | 619 | @@ -322,6 +322,7 @@ to the detailed description below for a |
ae10e51e WD |
620 | -q, --quiet suppress non-error messages |
621 | --no-motd suppress daemon-mode MOTD (see caveat) | |
622 | -c, --checksum skip based on checksum, not mod-time & size | |
56522462 | 623 | + --checksum-updating sender updates .rsyncsums files |
ae10e51e WD |
624 | -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) |
625 | --no-OPTION turn off an implied OPTION (e.g. --no-D) | |
626 | -r, --recursive recurse into directories | |
99650e0d | 627 | @@ -518,9 +519,9 @@ uses a "quick check" that (by default) c |
ae10e51e WD |
628 | of last modification match between the sender and receiver. This option |
629 | changes this to compare a 128-bit MD4 checksum for each file that has a | |
630 | matching size. Generating the checksums means that both sides will expend | |
631 | -a lot of disk I/O reading all the data in the files in the transfer (and | |
632 | -this is prior to any reading that will be done to transfer changed files), | |
633 | -so this can slow things down significantly. | |
634 | +a lot of disk I/O reading the data in all the files in the transfer, so | |
635 | +this can slow things down significantly (and this is prior to any reading | |
636 | +that will be done to transfer the files that have changed). | |
637 | ||
638 | The sending side generates its checksums while it is doing the file-system | |
639 | scan that builds the list of the available files. The receiver generates | |
99650e0d | 640 | @@ -528,12 +529,42 @@ its checksums when it is scanning for ch |
ae10e51e WD |
641 | file that has the same size as the corresponding sender's file: files with |
642 | either a changed size or a changed checksum are selected for transfer. | |
643 | ||
644 | +Starting with version 3.0.0, the sending side will look for a checksum | |
645 | +summary file and use a pre-generated checksum that it reads out of the file | |
646 | +(as long as it matches the file's size and modified time). This allows a | |
647 | +server to support the --checksum option to clients without having to | |
648 | +recompute the checksums for each client. See the bf(--checksum-updating) | |
56522462 | 649 | +option for a way to have rsync create/update these checksum files. |
ae10e51e WD |
650 | + |
651 | Note that rsync always verifies that each em(transferred) file was | |
652 | correctly reconstructed on the receiving side by checking a whole-file | |
653 | checksum that is generated when as the file is transferred, but that | |
654 | automatic after-the-transfer verification has nothing to do with this | |
655 | option's before-the-transfer "Does this file need to be updated?" check. | |
656 | ||
657 | +dit(bf(--checksum-updating)) This option tells the sending side to create | |
658 | +and/or update per-directory checksum files that are used by the | |
56522462 WD |
659 | +bf(--checksum) option. The file that is updated is named .rsyncsums. If |
660 | +pre-transfer checksums are not being computed, this option has no effect. | |
ae10e51e WD |
661 | + |
662 | +The checksum files stores the computed checksum, last-known size, | |
663 | +modification time, and name for each file in the current directory. If a | |
664 | +later transfer finds that a file matches its prior size and modification | |
665 | +time, the checksum is assumed to still be correct. Otherwise it is | |
666 | +recomputed and udpated in the file. | |
667 | + | |
668 | +To avoid transferring the system's checksum files, you can use an exclude | |
56522462 | 669 | +(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use |
ae10e51e WD |
670 | +a popt alias. For instance, adding the following line in your ~/.popt file |
671 | +defines a bf(-cc) option that enables checksum updating and excludes the | |
672 | +checksum files: | |
673 | + | |
56522462 | 674 | +verb( rsync alias --cc --checksum-updating --exclude=.rsyncsums) |
ae10e51e WD |
675 | + |
676 | +An rsync daemon does not allow the client to control this setting, so see | |
677 | +the "checksum updating" daemon config option for information on how to make | |
678 | +a daemon maintain these checksum files. | |
679 | + | |
680 | dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick | |
681 | way of saying you want recursion and want to preserve almost | |
682 | everything (with -H being a notable omission). | |
683 | --- old/rsyncd.conf.yo | |
684 | +++ new/rsyncd.conf.yo | |
56522462 | 685 | @@ -198,6 +198,20 @@ locking on this file to ensure that the |
ae10e51e WD |
686 | exceeded for the modules sharing the lock file. |
687 | The default is tt(/var/run/rsyncd.lock). | |
688 | ||
689 | +dit(bf(checksum updating)) This option tells rsync to update/create the | |
690 | +checksum information in the per-directory checksum files when users copy | |
691 | +files using the bf(--checksum) option. Any file that has changed since it | |
692 | +was last checksummed (or is not mentioned) has its data updated in the | |
56522462 | 693 | +.rsyncsums file. |
ae10e51e WD |
694 | + |
695 | +Note that this updating will occur even if the module is listed as being | |
696 | +read-only. If you want to hide these files (and you will almost always | |
56522462 | 697 | +want to do), add ".rsyncsums" to the module's exclude setting. |
ae10e51e WD |
698 | + |
699 | +Note also that the client's command-line option, bf(--checksum-updating), | |
700 | +has no effect on a daemon. A daemon will only update/create checksum files | |
701 | +if this config option is true. | |
702 | + | |
703 | dit(bf(read only)) The "read only" option determines whether clients | |
704 | will be able to upload files or not. If "read only" is true then any | |
705 | attempted uploads will fail. If "read only" is false then uploads will | |
7200c744 WD |
706 | --- old/support/rsyncsums |
707 | +++ new/support/rsyncsums | |
fcd80ec3 | 708 | @@ -0,0 +1,184 @@ |
7200c744 WD |
709 | +#!/usr/bin/perl -w |
710 | +use strict; | |
711 | + | |
712 | +use Getopt::Long; | |
713 | +use Cwd qw(abs_path cwd); | |
714 | +use Digest::MD4; | |
715 | +use Digest::MD5; | |
716 | + | |
717 | +our $SUMS_FILE = '.rsyncsums'; | |
718 | + | |
719 | +our($recurse_opt, $force_reading, $help_opt); | |
720 | +our $verbosity = 0; | |
721 | + | |
722 | +&Getopt::Long::Configure('bundling'); | |
723 | +&usage if !&GetOptions( | |
724 | + 'recurse|r' => \$recurse_opt, | |
725 | + 'force|f' => \$force_reading, | |
726 | + 'verbose|v+' => \$verbosity, | |
727 | + 'help|h' => \$help_opt, | |
728 | +) || $help_opt; | |
729 | + | |
730 | +my $start_dir = cwd(); | |
731 | + | |
732 | +my @dirs = @ARGV; | |
733 | +@dirs = '.' unless @dirs; | |
734 | +foreach (@dirs) { | |
735 | + $_ = abs_path($_); | |
736 | +} | |
737 | + | |
738 | +$| = 1; | |
739 | + | |
740 | +my $md4 = Digest::MD4->new; | |
741 | +my $md5 = Digest::MD5->new; | |
742 | + | |
743 | +while (@dirs) { | |
744 | + my $dir = shift @dirs; | |
745 | + | |
746 | + if (!chdir($dir)) { | |
747 | + warn "Unable to chdir to $dir: $!\n"; | |
748 | + next; | |
749 | + } | |
750 | + if (!opendir(DP, '.')) { | |
751 | + warn "Unable to opendir $dir: $!\n"; | |
752 | + next; | |
753 | + } | |
754 | + | |
755 | + if ($verbosity) { | |
756 | + my $reldir = $dir; | |
757 | + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; | |
758 | + print "$reldir ... "; | |
759 | + } | |
760 | + | |
761 | + my $sums_mtime = (stat($SUMS_FILE))[9]; | |
762 | + my %cache; | |
7200c744 WD |
763 | + my @subdirs; |
764 | + my $cnt = 0; | |
765 | + my $latest_time = 0; | |
766 | + while (defined(my $fn = readdir(DP))) { | |
767 | + next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn; | |
768 | + if (-d _) { | |
769 | + push(@subdirs, "$dir/$fn"); | |
770 | + next; | |
771 | + } | |
772 | + next unless -f _; | |
773 | + | |
774 | + my($size,$mtime,$ctime) = (stat(_))[7,9,10]; | |
775 | + | |
776 | + $cache{$fn} = [ $size, $mtime, $ctime ]; | |
777 | + $cnt++; | |
778 | + | |
779 | + $latest_time = $mtime if $mtime > $latest_time; | |
780 | + $latest_time = $ctime if $ctime > $latest_time; | |
781 | + } | |
782 | + | |
783 | + closedir DP; | |
784 | + | |
785 | + unshift(@dirs, sort @subdirs) if $recurse_opt; | |
786 | + | |
787 | + if (!$cnt) { | |
788 | + if (defined $sums_mtime) { | |
789 | + print "(removed $SUMS_FILE) " if $verbosity; | |
790 | + unlink($SUMS_FILE); | |
791 | + } | |
792 | + print "empty\n" if $verbosity; | |
793 | + next; | |
794 | + } | |
795 | + | |
796 | + if (defined($sums_mtime) && $sums_mtime == $latest_time && !$force_reading) { | |
797 | + print "OK\n" if $verbosity; | |
798 | + next; | |
799 | + } | |
800 | + | |
801 | + if (open(FP, '+<', $SUMS_FILE)) { | |
802 | + while (<FP>) { | |
803 | + chomp; | |
804 | + my($sum4, $sum5, $size, $mtime, $ctime, $fn) = split(' ', $_, 6); | |
805 | + my $ref = $cache{$fn}; | |
806 | + if (defined $ref) { | |
807 | + if ($$ref[0] == $size | |
808 | + && $$ref[1] == $mtime && $$ref[2] == $ctime | |
809 | + && $sum4 !~ /=/ && $sum5 !~ /=/) { | |
810 | + $$ref[3] = $sum4; | |
811 | + $$ref[4] = $sum5; | |
812 | + $cnt--; | |
813 | + } else { | |
814 | + $$ref[3] = $$ref[4] = undef; | |
815 | + } | |
816 | + } else { | |
817 | + $cnt = -1; # Force rewrite due to removed line. | |
818 | + } | |
819 | + } | |
820 | + } else { | |
821 | + open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n"; | |
822 | + $cnt = -1; | |
823 | + } | |
824 | + | |
825 | + if ($cnt) { | |
826 | + print "updating\n" if $verbosity; | |
827 | + while (my($fn, $ref) = each %cache) { | |
828 | + next if defined $$ref[3] && defined $$ref[4]; | |
829 | + if (!open(IN, $fn)) { | |
830 | + print STDERR "Unable to read $fn: $!\n"; | |
831 | + delete $cache{$fn}; | |
832 | + next; | |
833 | + } | |
834 | + | |
835 | + my($size,$mtime,$ctime) = (stat(IN))[7,9,10]; | |
836 | + my($sum4, $sum5); | |
837 | + | |
838 | + while (1) { | |
839 | + while (sysread(IN, $_, 64*1024)) { | |
840 | + $md4->add($_); | |
841 | + $md5->add($_); | |
842 | + } | |
843 | + $sum4 = $md4->hexdigest; | |
844 | + $sum5 = $md5->hexdigest; | |
edf38a9d WD |
845 | + print " $sum4 $sum5" if $verbosity > 2; |
846 | + print " $fn\n" if $verbosity > 1; | |
7200c744 WD |
847 | + my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10]; |
848 | + last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2; | |
849 | + $size = $size2; | |
850 | + $mtime = $mtime2; | |
851 | + $ctime = $ctime2; | |
852 | + sysseek(IN, 0, 0); | |
853 | + } | |
854 | + | |
855 | + close IN; | |
856 | + | |
857 | + $cache{$fn} = [ $size, $mtime, $ctime, $sum4, $sum5 ]; | |
858 | + } | |
859 | + | |
860 | + $latest_time = 0; | |
861 | + seek(FP, 0, 0); | |
862 | + foreach my $fn (sort keys %cache) { | |
863 | + my $ref = $cache{$fn}; | |
864 | + my($size, $mtime, $ctime, $sum4, $sum5) = @$ref; | |
53243b17 | 865 | + printf FP '%s %s %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn; |
7200c744 WD |
866 | + |
867 | + $latest_time = $mtime if $mtime > $latest_time; | |
868 | + $latest_time = $ctime if $ctime > $latest_time; | |
869 | + } | |
870 | + truncate(FP, tell(FP)); | |
871 | + } else { | |
872 | + print "OK.\n" if $verbosity; | |
873 | + } | |
874 | + | |
875 | + close FP; | |
876 | + | |
877 | + utime $latest_time, $latest_time, $SUMS_FILE; | |
878 | +} | |
879 | + | |
880 | +sub usage | |
881 | +{ | |
882 | + die <<EOT; | |
883 | +Usage: rsyncsums [OPTIONS] [DIRS] | |
884 | + | |
885 | +Options: | |
886 | + -r, --recurse Update $SUMS_FILE files in subdirectories too. | |
887 | + -f, --force Force the reading of an $SUMS_FILE file that looks to be | |
888 | + up-to-date. (Useful for weeding out old entries.) | |
889 | + -v, --verbose Mention what we're doing. Repeat for more info. | |
890 | + -h, --help Display this help message. | |
891 | +EOT | |
892 | +} |