Commit | Line | Data |
---|---|---|
f9df736a | 1 | Optimize the --checksum option using externally created .rsyncsums files. |
213d4328 | 2 | |
f9df736a WD |
3 | This adds a new option, --sumfiles=MODE, that allows you to use a cache of |
4 | checksums when performing a --checksum transfer. These checksum files | |
5 | (.rsyncsums) must be created by some other process -- see the perl script, | |
6 | rsyncsums, in the support dir for one way. | |
213d4328 | 7 | |
f9df736a WD |
8 | This option can be particularly helpful to a public mirror that wants to |
9 | pre-compute their .rsyncsums files, set the "checksum files = strict" option | |
10 | in their daemon config file, and thus make it quite efficient for a client | |
11 | rsync to make use of the --checksum option on their server. | |
213d4328 WD |
12 | |
13 | To use this patch, run these commands for a successful build: | |
14 | ||
cc3e685d | 15 | patch -p1 <patches/checksum-reading.diff |
213d4328 WD |
16 | ./configure (optional if already run) |
17 | make | |
18 | ||
f9df736a | 19 | diff --git a/checksum.c b/checksum.c |
fc557362 | 20 | index 811b5b6..f418dc2 100644 |
f9df736a WD |
21 | --- a/checksum.c |
22 | +++ b/checksum.c | |
fc557362 | 23 | @@ -98,7 +98,7 @@ void get_checksum2(char *buf, int32 len, char *sum) |
f9df736a WD |
24 | } |
25 | } | |
26 | ||
27 | -void file_checksum(char *fname, char *sum, OFF_T size) | |
28 | +void file_checksum(const char *fname, OFF_T size, char *sum) | |
29 | { | |
30 | struct map_struct *buf; | |
31 | OFF_T i, len = size; | |
32 | diff --git a/clientserver.c b/clientserver.c | |
fc557362 | 33 | index b6afe00..2681f33 100644 |
f9df736a WD |
34 | --- a/clientserver.c |
35 | +++ b/clientserver.c | |
c0c7984e | 36 | @@ -42,6 +42,8 @@ extern int numeric_ids; |
f9df736a WD |
37 | extern int filesfrom_fd; |
38 | extern int remote_protocol; | |
39 | extern int protocol_version; | |
40 | +extern int always_checksum; | |
41 | +extern int checksum_files; | |
42 | extern int io_timeout; | |
43 | extern int no_detach; | |
44 | extern int write_batch; | |
fc557362 | 45 | @@ -868,6 +870,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char |
c0c7984e | 46 | } else if (am_root < 0) /* Treat --fake-super from client as --super. */ |
f9df736a WD |
47 | am_root = 2; |
48 | ||
49 | + checksum_files = always_checksum ? lp_checksum_files(i) | |
50 | + : CSF_IGNORE_FILES; | |
51 | + | |
52 | if (filesfrom_fd == 0) | |
53 | filesfrom_fd = f_in; | |
54 | ||
cc3e685d | 55 | diff --git a/flist.c b/flist.c |
fc557362 | 56 | index 09b4fc5..8a42d80 100644 |
cc3e685d WD |
57 | --- a/flist.c |
58 | +++ b/flist.c | |
fc557362 WD |
59 | @@ -22,6 +22,7 @@ |
60 | ||
61 | #include "rsync.h" | |
62 | #include "ifuncs.h" | |
63 | +#include "itypes.h" | |
64 | #include "rounding.h" | |
65 | #include "inums.h" | |
66 | #include "io.h" | |
67 | @@ -33,6 +34,7 @@ extern int am_sender; | |
68 | extern int am_generator; | |
f9df736a | 69 | extern int inc_recurse; |
f9df736a WD |
70 | extern int always_checksum; |
71 | +extern int basis_dir_cnt; | |
72 | extern int module_id; | |
73 | extern int ignore_errors; | |
74 | extern int numeric_ids; | |
fc557362 | 75 | @@ -61,6 +63,7 @@ extern int file_extra_cnt; |
f9df736a WD |
76 | extern int ignore_perishable; |
77 | extern int non_perishable_cnt; | |
78 | extern int prune_empty_dirs; | |
79 | +extern int checksum_files; | |
80 | extern int copy_links; | |
81 | extern int copy_unsafe_links; | |
82 | extern int protocol_version; | |
fc557362 WD |
83 | @@ -71,6 +74,7 @@ extern int sender_symlink_iconv; |
84 | extern int output_needs_newline; | |
85 | extern int sender_keeps_checksum; | |
f9df736a WD |
86 | extern int unsort_ndx; |
87 | +extern char *basis_dir[]; | |
88 | extern struct stats stats; | |
89 | extern char *filesfrom_host; | |
fc557362 WD |
90 | extern char *usermap, *groupmap; |
91 | @@ -87,6 +91,12 @@ extern int filesfrom_convert; | |
f9df736a WD |
92 | extern iconv_t ic_send, ic_recv; |
93 | #endif | |
94 | ||
95 | +#define RSYNCSUMS_FILE ".rsyncsums" | |
c0c7984e | 96 | +#define RSYNCSUMS_LEN (sizeof RSYNCSUMS_FILE-1) |
f9df736a WD |
97 | + |
98 | +#define CLEAN_STRIP_ROOT (1<<0) | |
99 | +#define CLEAN_KEEP_LAST (1<<1) | |
100 | + | |
101 | #define PTR_SIZE (sizeof (struct file_struct *)) | |
102 | ||
103 | int io_error; | |
fc557362 WD |
104 | @@ -127,7 +137,11 @@ static char tmp_sum[MAX_DIGEST_LEN]; |
105 | static char empty_sum[MAX_DIGEST_LEN]; | |
213d4328 | 106 | static int flist_count_offset; /* for --delete --progress */ |
213d4328 | 107 | |
f9df736a WD |
108 | -static void flist_sort_and_clean(struct file_list *flist, int strip_root); |
109 | +static struct csum_cache { | |
110 | + struct file_list *flist; | |
111 | +} *csum_cache = NULL; | |
112 | + | |
113 | +static void flist_sort_and_clean(struct file_list *flist, int flags); | |
213d4328 | 114 | static void output_flist(struct file_list *flist); |
f9df736a WD |
115 | |
116 | void init_flist(void) | |
fc557362 | 117 | @@ -342,6 +356,238 @@ static void flist_done_allocating(struct file_list *flist) |
213d4328 WD |
118 | flist->pool_boundary = ptr; |
119 | } | |
120 | ||
f9df736a WD |
121 | +void reset_checksum_cache() |
122 | +{ | |
123 | + int slot, slots = am_sender ? 1 : basis_dir_cnt + 1; | |
124 | + | |
125 | + if (!csum_cache) { | |
126 | + csum_cache = new_array0(struct csum_cache, slots); | |
127 | + if (!csum_cache) | |
128 | + out_of_memory("reset_checksum_cache"); | |
129 | + } | |
130 | + | |
131 | + for (slot = 0; slot < slots; slot++) { | |
132 | + struct file_list *flist = csum_cache[slot].flist; | |
133 | + | |
134 | + if (flist) { | |
135 | + /* Reset the pool memory and empty the file-list array. */ | |
136 | + pool_free_old(flist->file_pool, | |
137 | + pool_boundary(flist->file_pool, 0)); | |
138 | + flist->used = 0; | |
139 | + } else | |
140 | + flist = csum_cache[slot].flist = flist_new(FLIST_TEMP, "reset_checksum_cache"); | |
141 | + | |
142 | + flist->low = 0; | |
143 | + flist->high = -1; | |
144 | + flist->next = NULL; | |
145 | + } | |
146 | +} | |
147 | + | |
c0c7984e | 148 | +/* The basename_len count is the length of the basename + 1 for the '\0'. */ |
f9df736a WD |
149 | +static int add_checksum(struct file_list *flist, const char *dirname, |
150 | + const char *basename, int basename_len, OFF_T file_length, | |
151 | + time_t mtime, uint32 ctime, uint32 inode, | |
152 | + const char *sum) | |
213d4328 WD |
153 | +{ |
154 | + struct file_struct *file; | |
155 | + int alloc_len, extra_len; | |
156 | + char *bp; | |
157 | + | |
c0c7984e WD |
158 | + if (basename_len == RSYNCSUMS_LEN+1 && *basename == '.' |
159 | + && strcmp(basename, RSYNCSUMS_FILE) == 0) | |
213d4328 WD |
160 | + return 0; |
161 | + | |
f9df736a WD |
162 | + /* "2" is for a 32-bit ctime num and an 32-bit inode num. */ |
163 | + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2) | |
213d4328 WD |
164 | + * EXTRA_LEN; |
165 | +#if EXTRA_ROUNDING > 0 | |
166 | + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) | |
167 | + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; | |
168 | +#endif | |
f9df736a WD |
169 | + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len; |
170 | + bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum"); | |
213d4328 WD |
171 | + |
172 | + memset(bp, 0, extra_len + FILE_STRUCT_LEN); | |
173 | + bp += extra_len; | |
174 | + file = (struct file_struct *)bp; | |
175 | + bp += FILE_STRUCT_LEN; | |
176 | + | |
f9df736a | 177 | + memcpy(bp, basename, basename_len); |
213d4328 WD |
178 | + |
179 | + file->mode = S_IFREG; | |
180 | + file->modtime = mtime; | |
181 | + file->len32 = (uint32)file_length; | |
182 | + if (file_length > 0xFFFFFFFFu) { | |
183 | + file->flags |= FLAG_LENGTH64; | |
184 | + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); | |
185 | + } | |
186 | + file->dirname = dirname; | |
f9df736a WD |
187 | + F_CTIME(file) = ctime; |
188 | + F_INODE(file) = inode; | |
213d4328 WD |
189 | + bp = F_SUM(file); |
190 | + memcpy(bp, sum, checksum_len); | |
191 | + | |
f9df736a WD |
192 | + flist_expand(flist, 1); |
193 | + flist->files[flist->used++] = file; | |
213d4328 | 194 | + |
f9df736a | 195 | + flist->sorted = flist->files; |
213d4328 WD |
196 | + |
197 | + return 1; | |
198 | +} | |
199 | + | |
f9df736a WD |
200 | +/* The "dirname" arg's data must remain unchanged during the lifespan of |
201 | + * the created csum_cache[].flist object because we use it directly. */ | |
202 | +static void read_checksums(int slot, struct file_list *flist, const char *dirname) | |
213d4328 WD |
203 | +{ |
204 | + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; | |
213d4328 | 205 | + FILE *fp; |
f9df736a WD |
206 | + char *cp; |
207 | + int len, i; | |
208 | + time_t mtime; | |
209 | + OFF_T file_length; | |
210 | + uint32 ctime, inode; | |
211 | + int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0; | |
213d4328 | 212 | + |
c0c7984e | 213 | + if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN)) |
213d4328 WD |
214 | + return; |
215 | + if (dlen) | |
216 | + fbuf[dlen++] = '/'; | |
217 | + else | |
218 | + dirname = NULL; | |
f9df736a WD |
219 | + strlcpy(fbuf+dlen, RSYNCSUMS_FILE, sizeof fbuf - dlen); |
220 | + if (slot) { | |
221 | + pathjoin(line, sizeof line, basis_dir[slot-1], fbuf); | |
222 | + cp = line; | |
223 | + } else | |
224 | + cp = fbuf; | |
225 | + if (!(fp = fopen(cp, "r"))) | |
213d4328 WD |
226 | + return; |
227 | + | |
228 | + while (fgets(line, sizeof line, fp)) { | |
229 | + cp = line; | |
230 | + if (protocol_version >= 30) { | |
231 | + char *alt_sum = cp; | |
232 | + if (*cp == '=') | |
233 | + while (*++cp == '=') {} | |
234 | + else | |
235 | + while (isXDigit(cp)) cp++; | |
236 | + if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ') | |
237 | + break; | |
238 | + while (*++cp == ' ') {} | |
239 | + } | |
240 | + | |
241 | + if (*cp == '=') { | |
242 | + continue; | |
243 | + } else { | |
244 | + for (i = 0; i < checksum_len*2; i++, cp++) { | |
245 | + int x; | |
246 | + if (isXDigit(cp)) { | |
247 | + if (isDigit(cp)) | |
248 | + x = *cp - '0'; | |
249 | + else | |
250 | + x = (*cp & 0xF) + 9; | |
251 | + } else { | |
252 | + cp = ""; | |
253 | + break; | |
254 | + } | |
255 | + if (i & 1) | |
256 | + sum[i/2] |= x; | |
257 | + else | |
258 | + sum[i/2] = x << 4; | |
259 | + } | |
260 | + } | |
261 | + if (*cp != ' ') | |
262 | + break; | |
263 | + while (*++cp == ' ') {} | |
264 | + | |
265 | + if (protocol_version < 30) { | |
266 | + char *alt_sum = cp; | |
267 | + if (*cp == '=') | |
268 | + while (*++cp == '=') {} | |
269 | + else | |
270 | + while (isXDigit(cp)) cp++; | |
271 | + if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ') | |
272 | + break; | |
273 | + while (*++cp == ' ') {} | |
274 | + } | |
275 | + | |
276 | + file_length = 0; | |
277 | + while (isDigit(cp)) | |
278 | + file_length = file_length * 10 + *cp++ - '0'; | |
279 | + if (*cp != ' ') | |
280 | + break; | |
281 | + while (*++cp == ' ') {} | |
282 | + | |
283 | + mtime = 0; | |
284 | + while (isDigit(cp)) | |
285 | + mtime = mtime * 10 + *cp++ - '0'; | |
286 | + if (*cp != ' ') | |
287 | + break; | |
288 | + while (*++cp == ' ') {} | |
289 | + | |
f9df736a | 290 | + ctime = 0; |
213d4328 | 291 | + while (isDigit(cp)) |
f9df736a | 292 | + ctime = ctime * 10 + *cp++ - '0'; |
213d4328 WD |
293 | + if (*cp != ' ') |
294 | + break; | |
295 | + while (*++cp == ' ') {} | |
296 | + | |
f9df736a | 297 | + inode = 0; |
213d4328 | 298 | + while (isDigit(cp)) |
f9df736a | 299 | + inode = inode * 10 + *cp++ - '0'; |
213d4328 WD |
300 | + if (*cp != ' ') |
301 | + break; | |
302 | + while (*++cp == ' ') {} | |
303 | + | |
304 | + len = strlen(cp); | |
305 | + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r')) | |
306 | + len--; | |
307 | + if (!len) | |
308 | + break; | |
309 | + cp[len++] = '\0'; /* len now counts the null */ | |
310 | + if (strchr(cp, '/')) | |
311 | + break; | |
312 | + if (len > MAXPATHLEN) | |
313 | + continue; | |
314 | + | |
315 | + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); | |
316 | + | |
f9df736a WD |
317 | + add_checksum(flist, dirname, cp, len, file_length, |
318 | + mtime, ctime, inode, | |
319 | + sum); | |
213d4328 WD |
320 | + } |
321 | + fclose(fp); | |
322 | + | |
f9df736a WD |
323 | + flist_sort_and_clean(flist, CLEAN_KEEP_LAST); |
324 | +} | |
325 | + | |
326 | +void get_cached_checksum(int slot, const char *fname, struct file_struct *file, | |
327 | + STRUCT_STAT *stp, char *sum_buf) | |
328 | +{ | |
329 | + struct file_list *flist = csum_cache[slot].flist; | |
330 | + int j; | |
331 | + | |
332 | + if (!flist->next) { | |
333 | + flist->next = cur_flist; /* next points from checksum flist to file flist */ | |
334 | + read_checksums(slot, flist, file->dirname); | |
335 | + } | |
336 | + | |
337 | + if ((j = flist_find(flist, file)) >= 0) { | |
338 | + struct file_struct *fp = flist->sorted[j]; | |
339 | + | |
340 | + if (F_LENGTH(fp) == stp->st_size | |
341 | + && fp->modtime == stp->st_mtime | |
342 | + && (checksum_files & CSF_LAX | |
343 | + || (F_CTIME(fp) == (uint32)stp->st_ctime | |
344 | + && F_INODE(fp) == (uint32)stp->st_ino))) { | |
345 | + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); | |
346 | + return; | |
347 | + } | |
348 | + } | |
349 | + | |
350 | + file_checksum(fname, stp->st_size, sum_buf); | |
213d4328 WD |
351 | +} |
352 | + | |
cbdf862c WD |
353 | /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's |
354 | * F_PATHNAME(), or (2) "NULL, dir, dirlen" to chdir() to the supplied dir, | |
355 | * with dir == NULL taken to be the starting directory, and dirlen < 0 | |
fc557362 | 356 | @@ -1105,7 +1351,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, |
213d4328 WD |
357 | STRUCT_STAT *stp, int flags, int filter_level) |
358 | { | |
359 | static char *lastdir; | |
360 | - static int lastdir_len = -1; | |
361 | + static int lastdir_len = -2; | |
362 | struct file_struct *file; | |
363 | char thisname[MAXPATHLEN]; | |
364 | char linkname[MAXPATHLEN]; | |
fc557362 | 365 | @@ -1251,9 +1497,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, |
213d4328 WD |
366 | memcpy(lastdir, thisname, len); |
367 | lastdir[len] = '\0'; | |
368 | lastdir_len = len; | |
f9df736a WD |
369 | + if (checksum_files && am_sender && flist) |
370 | + reset_checksum_cache(); | |
213d4328 WD |
371 | } |
372 | - } else | |
373 | + } else { | |
374 | basename = thisname; | |
f9df736a | 375 | + if (checksum_files && am_sender && flist && lastdir_len == -2) { |
213d4328 | 376 | + lastdir_len = -1; |
f9df736a | 377 | + reset_checksum_cache(); |
213d4328 WD |
378 | + } |
379 | + } | |
380 | basename_len = strlen(basename) + 1; /* count the '\0' */ | |
381 | ||
382 | #ifdef SUPPORT_LINKS | |
fc557362 WD |
383 | @@ -1267,11 +1520,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, |
384 | extra_len += EXTRA_LEN; | |
213d4328 WD |
385 | #endif |
386 | ||
fc557362 | 387 | - if (always_checksum && am_sender && S_ISREG(st.st_mode)) { |
213d4328 | 388 | - file_checksum(thisname, tmp_sum, st.st_size); |
fc557362 WD |
389 | - if (sender_keeps_checksum) |
390 | - extra_len += SUM_EXTRA_CNT * EXTRA_LEN; | |
391 | - } | |
392 | + if (sender_keeps_checksum && S_ISREG(st.st_mode)) | |
393 | + extra_len += SUM_EXTRA_CNT * EXTRA_LEN; | |
394 | ||
395 | #if EXTRA_ROUNDING > 0 | |
396 | if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) | |
397 | @@ -1347,8 +1597,14 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, | |
398 | return NULL; | |
399 | } | |
213d4328 | 400 | |
fc557362 WD |
401 | - if (sender_keeps_checksum && S_ISREG(st.st_mode)) |
402 | - memcpy(F_SUM(file), tmp_sum, checksum_len); | |
213d4328 | 403 | + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { |
f9df736a WD |
404 | + if (flist && checksum_files) |
405 | + get_cached_checksum(0, thisname, file, &st, tmp_sum); | |
406 | + else | |
407 | + file_checksum(thisname, st.st_size, tmp_sum); | |
fc557362 WD |
408 | + if (sender_keeps_checksum) |
409 | + memcpy(F_SUM(file), tmp_sum, checksum_len); | |
213d4328 | 410 | + } |
6e9495c7 | 411 | |
fc557362 WD |
412 | if (unsort_ndx) |
413 | F_NDX(file) = stats.num_dirs; | |
414 | @@ -2462,7 +2718,7 @@ struct file_list *recv_file_list(int f) | |
415 | flist_eof = 1; | |
416 | } | |
f9df736a WD |
417 | |
418 | - flist_sort_and_clean(flist, relative_paths); | |
419 | + flist_sort_and_clean(flist, relative_paths ? CLEAN_STRIP_ROOT : 0); | |
420 | ||
421 | if (protocol_version < 30) { | |
422 | /* Recv the io_error flag */ | |
fc557362 | 423 | @@ -2683,7 +2939,7 @@ void flist_free(struct file_list *flist) |
f9df736a WD |
424 | |
425 | /* This routine ensures we don't have any duplicate names in our file list. | |
426 | * duplicate names can cause corruption because of the pipelining. */ | |
427 | -static void flist_sort_and_clean(struct file_list *flist, int strip_root) | |
428 | +static void flist_sort_and_clean(struct file_list *flist, int flags) | |
429 | { | |
430 | char fbuf[MAXPATHLEN]; | |
431 | int i, prev_i; | |
fc557362 | 432 | @@ -2734,7 +2990,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) |
f9df736a WD |
433 | /* If one is a dir and the other is not, we want to |
434 | * keep the dir because it might have contents in the | |
435 | * list. Otherwise keep the first one. */ | |
436 | - if (S_ISDIR(file->mode)) { | |
437 | + if (S_ISDIR(file->mode) || flags & CLEAN_KEEP_LAST) { | |
438 | struct file_struct *fp = flist->sorted[j]; | |
439 | if (!S_ISDIR(fp->mode)) | |
440 | keep = i, drop = j; | |
fc557362 | 441 | @@ -2750,8 +3006,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) |
f9df736a WD |
442 | } else |
443 | keep = j, drop = i; | |
444 | ||
445 | - if (!am_sender) { | |
fc557362 | 446 | - if (DEBUG_GTE(DUP, 1)) { |
f9df736a | 447 | + if (!am_sender || flags & CLEAN_KEEP_LAST) { |
fc557362 | 448 | + if (DEBUG_GTE(DUP, 1) && !(flags & CLEAN_KEEP_LAST)) { |
f9df736a WD |
449 | rprintf(FINFO, |
450 | "removing duplicate name %s from file list (%d)\n", | |
451 | f_name(file, fbuf), drop + flist->ndx_start); | |
fc557362 | 452 | @@ -2773,7 +3029,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) |
f9df736a WD |
453 | } |
454 | flist->high = prev_i; | |
455 | ||
456 | - if (strip_root) { | |
457 | + if (flags & CLEAN_STRIP_ROOT) { | |
458 | /* We need to strip off the leading slashes for relative | |
459 | * paths, but this must be done _after_ the sorting phase. */ | |
460 | for (i = flist->low; i <= flist->high; i++) { | |
461 | diff --git a/generator.c b/generator.c | |
fc557362 | 462 | index 12007a1..48a5062 100644 |
f9df736a WD |
463 | --- a/generator.c |
464 | +++ b/generator.c | |
fc557362 WD |
465 | @@ -53,6 +53,7 @@ extern int delete_after; |
466 | extern int missing_args; | |
f9df736a WD |
467 | extern int msgdone_cnt; |
468 | extern int ignore_errors; | |
469 | +extern int checksum_files; | |
470 | extern int remove_source_files; | |
471 | extern int delay_updates; | |
472 | extern int update_only; | |
fc557362 | 473 | @@ -522,7 +523,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre |
f9df736a WD |
474 | |
475 | ||
476 | /* Perform our quick-check heuristic for determining if a file is unchanged. */ | |
477 | -int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) | |
478 | +int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot) | |
479 | { | |
480 | if (st->st_size != F_LENGTH(file)) | |
481 | return 0; | |
fc557362 | 482 | @@ -531,7 +532,10 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) |
f9df736a WD |
483 | of the file time to determine whether to sync */ |
484 | if (always_checksum > 0 && S_ISREG(st->st_mode)) { | |
485 | char sum[MAX_DIGEST_LEN]; | |
486 | - file_checksum(fn, sum, st->st_size); | |
487 | + if (checksum_files && slot >= 0) | |
488 | + get_cached_checksum(slot, fn, file, st, sum); | |
489 | + else | |
490 | + file_checksum(fn, st->st_size, sum); | |
491 | return memcmp(sum, F_SUM(file), checksum_len) == 0; | |
492 | } | |
493 | ||
fc557362 | 494 | @@ -795,7 +799,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, |
f9df736a WD |
495 | match_level = 1; |
496 | /* FALL THROUGH */ | |
497 | case 1: | |
498 | - if (!unchanged_file(cmpbuf, file, &sxp->st)) | |
499 | + if (!unchanged_file(cmpbuf, file, &sxp->st, j+1)) | |
500 | continue; | |
501 | best_match = j; | |
502 | match_level = 2; | |
fc557362 | 503 | @@ -1074,7 +1078,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, |
c0c7984e WD |
504 | * --ignore-non-existing, daemon exclude, or mkdir failure. */ |
505 | static struct file_struct *skip_dir = NULL; | |
f9df736a WD |
506 | static struct file_list *fuzzy_dirlist = NULL; |
507 | - static int need_fuzzy_dirlist = 0; | |
508 | + static int need_new_dirscan = 0; | |
509 | struct file_struct *fuzzy_file = NULL; | |
510 | int fd = -1, f_copy = -1; | |
511 | stat_x sx, real_sx; | |
fc557362 | 512 | @@ -1158,8 +1162,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, |
f9df736a WD |
513 | flist_free(fuzzy_dirlist); |
514 | fuzzy_dirlist = NULL; | |
515 | } | |
516 | - if (fuzzy_basis) | |
517 | - need_fuzzy_dirlist = 1; | |
518 | + if (fuzzy_basis || checksum_files) | |
519 | + need_new_dirscan = 1; | |
520 | #ifdef SUPPORT_ACLS | |
521 | if (!preserve_perms) | |
522 | dflt_perms = default_perms_for_dir(dn); | |
fc557362 | 523 | @@ -1167,10 +1171,15 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, |
f9df736a WD |
524 | } |
525 | parent_dirname = dn; | |
526 | ||
527 | - if (need_fuzzy_dirlist && S_ISREG(file->mode)) { | |
528 | - strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); | |
529 | - fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); | |
530 | - need_fuzzy_dirlist = 0; | |
531 | + if (need_new_dirscan && S_ISREG(file->mode)) { | |
532 | + if (fuzzy_basis) { | |
533 | + strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); | |
534 | + fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); | |
535 | + } | |
536 | + if (checksum_files) { | |
537 | + reset_checksum_cache(); | |
538 | + } | |
539 | + need_new_dirscan = 0; | |
540 | } | |
541 | ||
542 | statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir); | |
fc557362 | 543 | @@ -1612,7 +1621,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, |
f9df736a WD |
544 | ; |
545 | else if (fnamecmp_type == FNAMECMP_FUZZY) | |
546 | ; | |
547 | - else if (unchanged_file(fnamecmp, file, &sx.st)) { | |
548 | + else if (unchanged_file(fnamecmp, file, &sx.st, fnamecmp_type == FNAMECMP_FNAME ? 0 : -1)) { | |
549 | if (partialptr) { | |
550 | do_unlink(partialptr); | |
551 | handle_partial_dir(partialptr, PDIR_DELETE); | |
552 | diff --git a/hlink.c b/hlink.c | |
fc557362 | 553 | index c9eb33a..6109266 100644 |
f9df736a WD |
554 | --- a/hlink.c |
555 | +++ b/hlink.c | |
fc557362 | 556 | @@ -413,7 +413,7 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname, |
f9df736a WD |
557 | } |
558 | break; | |
559 | } | |
560 | - if (!unchanged_file(cmpbuf, file, &alt_sx.st)) | |
561 | + if (!unchanged_file(cmpbuf, file, &alt_sx.st, j+1)) | |
562 | continue; | |
563 | statret = 1; | |
564 | if (unchanged_attrs(cmpbuf, file, &alt_sx)) | |
fc557362 WD |
565 | diff --git a/itypes.h b/itypes.h |
566 | index df34140..1bdf506 100644 | |
567 | --- a/itypes.h | |
568 | +++ b/itypes.h | |
569 | @@ -23,6 +23,12 @@ isDigit(const char *ptr) | |
213d4328 WD |
570 | } |
571 | ||
572 | static inline int | |
573 | +isXDigit(const char *ptr) | |
574 | +{ | |
575 | + return isxdigit(*(unsigned char *)ptr); | |
576 | +} | |
577 | + | |
578 | +static inline int | |
579 | isPrint(const char *ptr) | |
580 | { | |
581 | return isprint(*(unsigned char *)ptr); | |
f9df736a | 582 | diff --git a/loadparm.c b/loadparm.c |
fc557362 | 583 | index 8e48e6d..899d2b5 100644 |
f9df736a WD |
584 | --- a/loadparm.c |
585 | +++ b/loadparm.c | |
fc557362 WD |
586 | @@ -132,6 +132,7 @@ typedef struct { |
587 | /* NOTE: update this macro if the last char* variable changes! */ | |
588 | #define LOCAL_STRING_COUNT() (offsetof(local_vars, uid) / sizeof (char*) + 1) | |
f9df736a WD |
589 | |
590 | + int checksum_files; | |
591 | int max_connections; | |
592 | int max_verbosity; | |
593 | int syslog_facility; | |
fc557362 | 594 | @@ -204,6 +205,7 @@ static const all_vars Defaults = { |
f9df736a | 595 | /* temp_dir; */ NULL, |
fc557362 | 596 | /* uid; */ NULL, |
f9df736a WD |
597 | |
598 | + /* checksum_files; */ CSF_IGNORE_FILES, | |
599 | /* max_connections; */ 0, | |
600 | /* max_verbosity; */ 1, | |
601 | /* syslog_facility; */ LOG_DAEMON, | |
fc557362 WD |
602 | @@ -305,6 +307,13 @@ static struct enum_list enum_facilities[] = { |
603 | { -1, NULL } | |
604 | }; | |
f9df736a WD |
605 | |
606 | +static struct enum_list enum_csum_modes[] = { | |
607 | + { CSF_IGNORE_FILES, "none" }, | |
608 | + { CSF_LAX_MODE, "lax" }, | |
609 | + { CSF_STRICT_MODE, "strict" }, | |
610 | + { -1, NULL } | |
611 | +}; | |
fc557362 | 612 | + |
f9df736a | 613 | static struct parm_struct parm_table[] = |
fc557362 WD |
614 | { |
615 | {"address", P_STRING, P_GLOBAL,&Vars.g.bind_address, NULL,0}, | |
616 | @@ -315,6 +324,7 @@ static struct parm_struct parm_table[] = | |
617 | ||
618 | {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0}, | |
619 | {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0}, | |
620 | + {"checksum files", P_ENUM, P_LOCAL, &Vars.l.checksum_files, enum_csum_modes,0}, | |
621 | {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0}, | |
622 | {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0}, | |
623 | {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0}, | |
624 | @@ -419,6 +429,7 @@ FN_LOCAL_STRING(lp_secrets_file, secrets_file) | |
f9df736a WD |
625 | FN_LOCAL_STRING(lp_temp_dir, temp_dir) |
626 | FN_LOCAL_STRING(lp_uid, uid) | |
627 | ||
628 | +FN_LOCAL_INTEGER(lp_checksum_files, checksum_files) | |
629 | FN_LOCAL_INTEGER(lp_max_connections, max_connections) | |
630 | FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity) | |
631 | FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility) | |
632 | diff --git a/options.c b/options.c | |
fc557362 | 633 | index e7c6c61..2e110f3 100644 |
f9df736a WD |
634 | --- a/options.c |
635 | +++ b/options.c | |
fc557362 | 636 | @@ -112,6 +112,7 @@ size_t bwlimit_writemax = 0; |
f9df736a WD |
637 | int ignore_existing = 0; |
638 | int ignore_non_existing = 0; | |
639 | int need_messages_from_generator = 0; | |
640 | +int checksum_files = CSF_IGNORE_FILES; | |
641 | int max_delete = INT_MIN; | |
642 | OFF_T max_size = 0; | |
643 | OFF_T min_size = 0; | |
fc557362 | 644 | @@ -661,6 +662,7 @@ void usage(enum logcode F) |
f9df736a WD |
645 | rprintf(F," -q, --quiet suppress non-error messages\n"); |
646 | rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); | |
647 | rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); | |
648 | + rprintf(F," --sumfiles=MODE use .rsyncsums to speedup --checksum mode\n"); | |
649 | rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); | |
650 | rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); | |
651 | rprintf(F," -r, --recursive recurse into directories\n"); | |
fc557362 | 652 | @@ -798,7 +800,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, |
f9df736a WD |
653 | OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, |
654 | OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, | |
fc557362 WD |
655 | OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, |
656 | - OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, | |
657 | + OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_SUMFILES, | |
f9df736a WD |
658 | OPT_SERVER, OPT_REFUSED_BASE = 9000}; |
659 | ||
660 | static struct poptOption long_options[] = { | |
fc557362 | 661 | @@ -933,6 +935,7 @@ static struct poptOption long_options[] = { |
f9df736a WD |
662 | {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, |
663 | {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
664 | {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
665 | + {"sumfiles", 0, POPT_ARG_STRING, 0, OPT_SUMFILES, 0, 0 }, | |
666 | {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, | |
667 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, | |
668 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, | |
fc557362 | 669 | @@ -1630,6 +1633,23 @@ int parse_arguments(int *argc_p, const char ***argv_p) |
f9df736a WD |
670 | } |
671 | break; | |
672 | ||
673 | + case OPT_SUMFILES: | |
674 | + arg = poptGetOptArg(pc); | |
675 | + checksum_files = 0; | |
676 | + if (strcmp(arg, "lax") == 0) | |
677 | + checksum_files |= CSF_LAX_MODE; | |
678 | + else if (strcmp(arg, "strict") == 0) | |
679 | + checksum_files |= CSF_STRICT_MODE; | |
680 | + else if (strcmp(arg, "none") == 0) | |
681 | + checksum_files = CSF_IGNORE_FILES; | |
682 | + else { | |
683 | + snprintf(err_buf, sizeof err_buf, | |
684 | + "Invalid argument passed to --sumfiles (%s)\n", | |
685 | + arg); | |
686 | + return 0; | |
687 | + } | |
688 | + break; | |
689 | + | |
fc557362 WD |
690 | case OPT_INFO: |
691 | arg = poptGetOptArg(pc); | |
692 | parse_output_words(info_words, info_levels, arg, USER_PRIORITY); | |
693 | @@ -1830,6 +1850,9 @@ int parse_arguments(int *argc_p, const char ***argv_p) | |
f9df736a WD |
694 | } |
695 | #endif | |
696 | ||
c0c7984e | 697 | + if (!always_checksum) |
f9df736a WD |
698 | + checksum_files = CSF_IGNORE_FILES; |
699 | + | |
700 | if (write_batch && read_batch) { | |
701 | snprintf(err_buf, sizeof err_buf, | |
702 | "--write-batch and --read-batch can not be used together\n"); | |
703 | diff --git a/rsync.h b/rsync.h | |
fc557362 | 704 | index be7cf8a..ba8f3db 100644 |
f9df736a WD |
705 | --- a/rsync.h |
706 | +++ b/rsync.h | |
fc557362 | 707 | @@ -712,6 +712,10 @@ extern int xattrs_ndx; |
f9df736a WD |
708 | #define F_SUM(f) ((char*)OPT_EXTRA(f, LEN64_BUMP(f) + HLINK_BUMP(f) \ |
709 | + SUM_EXTRA_CNT - 1)) | |
710 | ||
711 | +/* These are only valid on an entry read from a checksum file. */ | |
712 | +#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->unum | |
713 | +#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->unum | |
714 | + | |
715 | /* Some utility defines: */ | |
716 | #define F_IS_ACTIVE(f) (f)->basename[0] | |
717 | #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED) | |
fc557362 | 718 | @@ -902,6 +906,13 @@ typedef struct { |
f9df736a WD |
719 | char fname[1]; /* has variable size */ |
720 | } relnamecache; | |
721 | ||
722 | +#define CSF_ENABLE (1<<1) | |
723 | +#define CSF_LAX (1<<2) | |
724 | + | |
725 | +#define CSF_IGNORE_FILES 0 | |
726 | +#define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX) | |
727 | +#define CSF_STRICT_MODE (CSF_ENABLE) | |
728 | + | |
729 | #include "byteorder.h" | |
730 | #include "lib/mdigest.h" | |
731 | #include "lib/wildmatch.h" | |
732 | diff --git a/rsync.yo b/rsync.yo | |
fc557362 | 733 | index 941f7a5..7aa62cf 100644 |
f9df736a WD |
734 | --- a/rsync.yo |
735 | +++ b/rsync.yo | |
fc557362 | 736 | @@ -323,6 +323,7 @@ to the detailed description below for a complete description. verb( |
f9df736a WD |
737 | -q, --quiet suppress non-error messages |
738 | --no-motd suppress daemon-mode MOTD (see caveat) | |
739 | -c, --checksum skip based on checksum, not mod-time & size | |
740 | + --sumfiles=MODE use .rsyncsums to speedup --checksum mode | |
741 | -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) | |
742 | --no-OPTION turn off an implied OPTION (e.g. --no-D) | |
743 | -r, --recursive recurse into directories | |
fc557362 | 744 | @@ -568,9 +569,9 @@ uses a "quick check" that (by default) checks if each file's size and time |
f9df736a WD |
745 | of last modification match between the sender and receiver. This option |
746 | changes this to compare a 128-bit MD4 checksum for each file that has a | |
747 | matching size. Generating the checksums means that both sides will expend | |
748 | -a lot of disk I/O reading all the data in the files in the transfer (and | |
749 | -this is prior to any reading that will be done to transfer changed files), | |
750 | -so this can slow things down significantly. | |
751 | +a lot of disk I/O reading the data in all the files in the transfer, so | |
752 | +this can slow things down significantly (and this is prior to any reading | |
753 | +that will be done to transfer the files that have changed). | |
754 | ||
755 | The sending side generates its checksums while it is doing the file-system | |
756 | scan that builds the list of the available files. The receiver generates | |
fc557362 | 757 | @@ -578,12 +579,44 @@ its checksums when it is scanning for changed files, and will checksum any |
f9df736a WD |
758 | file that has the same size as the corresponding sender's file: files with |
759 | either a changed size or a changed checksum are selected for transfer. | |
760 | ||
761 | +See also the bf(--sumfiles) option for a way to use cached checksum data. | |
762 | + | |
763 | Note that rsync always verifies that each em(transferred) file was | |
764 | correctly reconstructed on the receiving side by checking a whole-file | |
765 | checksum that is generated as the file is transferred, but that | |
766 | automatic after-the-transfer verification has nothing to do with this | |
767 | option's before-the-transfer "Does this file need to be updated?" check. | |
768 | ||
769 | +dit(bf(--sumfiles=MODE)) This option tells rsync to make use of any cached | |
770 | +checksum information it finds in per-directory .rsyncsums files when the | |
771 | +current transfer is using the bf(--checksum) option. If the checksum data | |
772 | +is up-to-date, it is used instead of recomputing it, saving both disk I/O | |
773 | +and CPU time. If the checksum data is missing or outdated, the checksum is | |
774 | +computed just as it would be if bf(--sumfiles) was not specified. | |
775 | + | |
776 | +The MODE value is either "lax", for relaxed checking (which compares size | |
777 | +and mtime), "strict" (which also compares ctime and inode), or "none" to | |
778 | +ignore any .rsyncsums files ("none" is the default). Rsync does not create | |
779 | +or update these files, but there is a perl script in the support directory | |
780 | +named "rsyncsums" that can be used for that. | |
781 | + | |
782 | +This option has no effect unless bf(--checksum, -c) was also specified. It | |
783 | +also only affects the current side of the transfer, so if you want the | |
784 | +remote side to parse its own .rsyncsums files, specify the option via the | |
785 | +bf(--rsync-path) option (e.g. "--rsync-path="rsync --sumfiles=lax"). | |
786 | + | |
787 | +To avoid transferring the system's checksum files, you can use an exclude | |
788 | +(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use | |
789 | +a popt alias. For instance, adding the following line in your ~/.popt file | |
790 | +defines a bf(--cc) option that enables lax checksum files and excludes the | |
791 | +checksum files: | |
792 | + | |
793 | +verb( rsync alias --cc -c --sumfiles=lax --exclude=.rsyncsums) | |
794 | + | |
795 | +An rsync daemon does not allow the client to control this setting, so see | |
796 | +the "checksum files" daemon parameter for information on how to make a | |
797 | +daemon use cached checksum data. | |
798 | + | |
799 | dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick | |
800 | way of saying you want recursion and want to preserve almost | |
801 | everything (with -H being a notable omission). | |
802 | diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo | |
fc557362 | 803 | index d4978cd..0fc98fd 100644 |
f9df736a WD |
804 | --- a/rsyncd.conf.yo |
805 | +++ b/rsyncd.conf.yo | |
fc557362 | 806 | @@ -292,6 +292,17 @@ locking on this file to ensure that the max connections limit is not |
f9df736a WD |
807 | exceeded for the modules sharing the lock file. |
808 | The default is tt(/var/run/rsyncd.lock). | |
809 | ||
e66d6d51 | 810 | +dit(bf(checksum files)) This parameter tells rsync to make use of any cached |
f9df736a WD |
811 | +checksum information it finds in per-directory .rsyncsums files when the |
812 | +current transfer is using the bf(--checksum) option. The value can be set | |
813 | +to either "lax", "strict", or "none" -- see the client's bf(--sumfiles) | |
814 | +option for what these choices do. | |
815 | + | |
816 | +Note also that the client's command-line option, bf(--sumfiles), has no | |
817 | +effect on a daemon. A daemon will only access checksum files if this | |
818 | +config option tells it to. See also the bf(exclude) directive for a way | |
819 | +to hide the .rsyncsums files from the user. | |
820 | + | |
e66d6d51 | 821 | dit(bf(read only)) This parameter determines whether clients |
f9df736a WD |
822 | will be able to upload files or not. If "read only" is true then any |
823 | attempted uploads will fail. If "read only" is false then uploads will | |
cc3e685d | 824 | diff --git a/support/rsyncsums b/support/rsyncsums |
f9df736a | 825 | new file mode 100755 |
fc557362 | 826 | index 0000000..ce03c80 |
cc3e685d WD |
827 | --- /dev/null |
828 | +++ b/support/rsyncsums | |
fc557362 WD |
829 | @@ -0,0 +1,201 @@ |
830 | +#!/usr/bin/perl -w | |
213d4328 WD |
831 | +use strict; |
832 | + | |
833 | +use Getopt::Long; | |
834 | +use Cwd qw(abs_path cwd); | |
835 | +use Digest::MD4; | |
836 | +use Digest::MD5; | |
837 | + | |
838 | +our $SUMS_FILE = '.rsyncsums'; | |
839 | + | |
840 | +&Getopt::Long::Configure('bundling'); | |
841 | +&usage if !&GetOptions( | |
213d4328 | 842 | + 'recurse|r' => \( my $recurse_opt ), |
f9df736a | 843 | + 'mode|m=s' => \( my $cmp_mode = 'strict' ), |
d1a75c9f | 844 | + 'check|c' => \( my $check_opt ), |
213d4328 WD |
845 | + 'verbose|v+' => \( my $verbosity = 0 ), |
846 | + 'help|h' => \( my $help_opt ), | |
847 | +); | |
f9df736a WD |
848 | +&usage if $help_opt || $cmp_mode !~ /^(lax|strict)$/; |
849 | + | |
850 | +my $ignore_ctime_and_inode = $cmp_mode eq 'lax' ? 0 : 1; | |
213d4328 WD |
851 | + |
852 | +my $start_dir = cwd(); | |
853 | + | |
854 | +my @dirs = @ARGV; | |
855 | +@dirs = '.' unless @dirs; | |
856 | +foreach (@dirs) { | |
857 | + $_ = abs_path($_); | |
858 | +} | |
859 | + | |
860 | +$| = 1; | |
861 | + | |
d1a75c9f WD |
862 | +my $exit_code = 0; |
863 | + | |
213d4328 WD |
864 | +my $md4 = Digest::MD4->new; |
865 | +my $md5 = Digest::MD5->new; | |
866 | + | |
867 | +while (@dirs) { | |
868 | + my $dir = shift @dirs; | |
869 | + | |
870 | + if (!chdir($dir)) { | |
871 | + warn "Unable to chdir to $dir: $!\n"; | |
872 | + next; | |
873 | + } | |
874 | + if (!opendir(DP, '.')) { | |
875 | + warn "Unable to opendir $dir: $!\n"; | |
876 | + next; | |
877 | + } | |
878 | + | |
d1a75c9f WD |
879 | + my $reldir = $dir; |
880 | + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; | |
213d4328 | 881 | + if ($verbosity) { |
213d4328 | 882 | + print "$reldir ... "; |
d1a75c9f | 883 | + print "\n" if $check_opt; |
213d4328 WD |
884 | + } |
885 | + | |
213d4328 | 886 | + my %cache; |
d1a75c9f WD |
887 | + my $f_cnt = 0; |
888 | + if (open(FP, '<', $SUMS_FILE)) { | |
889 | + while (<FP>) { | |
890 | + chomp; | |
891 | + my($sum4, $sum5, $size, $mtime, $ctime, $inode, $fn) = split(' ', $_, 7); | |
892 | + $cache{$fn} = [ 0, $sum4, $sum5, $size, $mtime, $ctime & 0xFFFFFFFF, $inode & 0xFFFFFFFF ]; | |
893 | + $f_cnt++; | |
894 | + } | |
895 | + close FP; | |
896 | + } | |
897 | + | |
213d4328 | 898 | + my @subdirs; |
d1a75c9f WD |
899 | + my $d_cnt = 0; |
900 | + my $update_cnt = 0; | |
213d4328 WD |
901 | + while (defined(my $fn = readdir(DP))) { |
902 | + next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn; | |
903 | + if (-d _) { | |
904 | + push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/; | |
905 | + next; | |
906 | + } | |
907 | + next unless -f _; | |
908 | + | |
909 | + my($size,$mtime,$ctime,$inode) = (stat(_))[7,9,10,1]; | |
f9df736a WD |
910 | + $ctime &= 0xFFFFFFFF; |
911 | + $inode &= 0xFFFFFFFF; | |
d1a75c9f | 912 | + my $ref = $cache{$fn}; |
d1a75c9f | 913 | + $d_cnt++; |
213d4328 | 914 | + |
d1a75c9f | 915 | + if (!$check_opt) { |
213d4328 | 916 | + if (defined $ref) { |
d1a75c9f WD |
917 | + $$ref[0] = 1; |
918 | + if ($$ref[3] == $size | |
919 | + && $$ref[4] == $mtime | |
920 | + && ($ignore_ctime_and_inode || ($$ref[5] == $ctime && $$ref[6] == $inode)) | |
921 | + && $$ref[1] !~ /=/ && $$ref[2] !~ /=/) { | |
922 | + next; | |
213d4328 | 923 | + } |
d1a75c9f WD |
924 | + } |
925 | + if (!$update_cnt++) { | |
926 | + print "UPDATING\n" if $verbosity; | |
213d4328 WD |
927 | + } |
928 | + } | |
213d4328 | 929 | + |
d1a75c9f WD |
930 | + if (!open(IN, $fn)) { |
931 | + print STDERR "Unable to read $fn: $!\n"; | |
932 | + if (defined $ref) { | |
213d4328 | 933 | + delete $cache{$fn}; |
d1a75c9f | 934 | + $f_cnt--; |
213d4328 | 935 | + } |
d1a75c9f WD |
936 | + next; |
937 | + } | |
213d4328 | 938 | + |
d1a75c9f WD |
939 | + my($sum4, $sum5); |
940 | + while (1) { | |
941 | + while (sysread(IN, $_, 64*1024)) { | |
942 | + $md4->add($_); | |
943 | + $md5->add($_); | |
213d4328 | 944 | + } |
d1a75c9f WD |
945 | + $sum4 = $md4->hexdigest; |
946 | + $sum5 = $md5->hexdigest; | |
947 | + print " $sum4 $sum5" if $verbosity > 2; | |
948 | + print " $fn" if $verbosity > 1; | |
949 | + my($size2,$mtime2,$ctime2,$inode2) = (stat(IN))[7,9,10,1]; | |
f9df736a WD |
950 | + $ctime2 &= 0xFFFFFFFF; |
951 | + $inode2 &= 0xFFFFFFFF; | |
d1a75c9f WD |
952 | + last if $size == $size2 && $mtime == $mtime2 |
953 | + && ($ignore_ctime_and_inode || ($ctime == $ctime2 && $inode == $inode2)); | |
954 | + $size = $size2; | |
955 | + $mtime = $mtime2; | |
956 | + $ctime = $ctime2; | |
957 | + $inode = $inode2; | |
958 | + sysseek(IN, 0, 0); | |
959 | + print " REREADING\n" if $verbosity > 1; | |
960 | + } | |
213d4328 | 961 | + |
d1a75c9f WD |
962 | + close IN; |
963 | + | |
964 | + if ($check_opt) { | |
965 | + my $dif; | |
966 | + if (!defined $ref) { | |
967 | + $dif = 'MISSING'; | |
968 | + } elsif ($sum4 ne $$ref[1] || $sum5 ne $$ref[2]) { | |
969 | + $dif = 'FAILED'; | |
970 | + } else { | |
971 | + print " OK\n" if $verbosity > 1; | |
972 | + next; | |
973 | + } | |
974 | + if ($verbosity < 2) { | |
975 | + print $verbosity ? ' ' : "$reldir/"; | |
976 | + print $fn; | |
213d4328 | 977 | + } |
d1a75c9f WD |
978 | + print " $dif\n"; |
979 | + $exit_code = 1; | |
980 | + } else { | |
981 | + print "\n" if $verbosity > 1; | |
f9df736a | 982 | + $cache{$fn} = [ 1, $sum4, $sum5, $size, $mtime, $ctime, $inode ]; |
d1a75c9f WD |
983 | + } |
984 | + } | |
985 | + | |
986 | + closedir DP; | |
213d4328 | 987 | + |
d1a75c9f WD |
988 | + unshift(@dirs, sort @subdirs) if $recurse_opt; |
989 | + | |
990 | + if ($check_opt) { | |
991 | + ; | |
992 | + } elsif ($d_cnt == 0) { | |
993 | + if ($f_cnt) { | |
994 | + print "(removed $SUMS_FILE) " if $verbosity; | |
995 | + unlink($SUMS_FILE); | |
213d4328 | 996 | + } |
d1a75c9f WD |
997 | + print "empty\n" if $verbosity; |
998 | + } elsif ($update_cnt || $d_cnt != $f_cnt) { | |
999 | + print "UPDATING\n" if $verbosity && !$update_cnt; | |
1000 | + open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n"; | |
213d4328 | 1001 | + |
213d4328 WD |
1002 | + foreach my $fn (sort keys %cache) { |
1003 | + my $ref = $cache{$fn}; | |
d1a75c9f WD |
1004 | + my($found, $sum4, $sum5, $size, $mtime, $ctime, $inode) = @$ref; |
1005 | + next unless $found; | |
213d4328 WD |
1006 | + printf FP '%s %s %10d %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $inode, $fn; |
1007 | + } | |
d1a75c9f | 1008 | + close FP; |
213d4328 WD |
1009 | + } else { |
1010 | + print "ok\n" if $verbosity; | |
1011 | + } | |
213d4328 WD |
1012 | +} |
1013 | + | |
d1a75c9f WD |
1014 | +exit $exit_code; |
1015 | + | |
213d4328 WD |
1016 | +sub usage |
1017 | +{ | |
1018 | + die <<EOT; | |
1019 | +Usage: rsyncsums [OPTIONS] [DIRS] | |
1020 | + | |
1021 | +Options: | |
1022 | + -r, --recurse Update $SUMS_FILE files in subdirectories too. | |
f9df736a WD |
1023 | + -m, --mode=MODE Compare entries in either "lax" or "strict" mode. Using |
1024 | + "lax" compares size and mtime, while "strict" additionally | |
1025 | + compares ctime and inode. Default: strict. | |
d1a75c9f | 1026 | + -c, --check Check if the checksums are right (doesn't update). |
213d4328 WD |
1027 | + -v, --verbose Mention what we're doing. Repeat for more info. |
1028 | + -h, --help Display this help message. | |
1029 | +EOT | |
1030 | +} |