| 1 | This adds a sender optimization feature that allows a cache of checksums |
| 2 | to be used when the client specifies the --checksum option, and creates |
| 3 | and/or updates the .rsyncsums files when --checksum-updating is |
| 4 | specified. |
| 5 | |
| 6 | To use this patch, run these commands for a successful build: |
| 7 | |
| 8 | patch -p1 <patches/checksum-updating.diff |
| 9 | ./configure (optional if already run) |
| 10 | make |
| 11 | |
| 12 | --- old/clientserver.c |
| 13 | +++ new/clientserver.c |
| 14 | @@ -37,6 +37,7 @@ extern int sanitize_paths; |
| 15 | extern int filesfrom_fd; |
| 16 | extern int remote_protocol; |
| 17 | extern int protocol_version; |
| 18 | +extern int checksum_updating; |
| 19 | extern int io_timeout; |
| 20 | extern int no_detach; |
| 21 | extern int default_af_hint; |
| 22 | @@ -634,6 +635,8 @@ static int rsync_module(int f_in, int f_ |
| 23 | else if (am_root < 0) /* Treat --fake-super from client as --super. */ |
| 24 | am_root = 2; |
| 25 | |
| 26 | + checksum_updating = lp_checksum_updating(i); |
| 27 | + |
| 28 | if (filesfrom_fd == 0) |
| 29 | filesfrom_fd = f_in; |
| 30 | |
| 31 | --- old/flist.c |
| 32 | +++ new/flist.c |
| 33 | @@ -25,6 +25,7 @@ |
| 34 | #include "io.h" |
| 35 | |
| 36 | extern int verbose; |
| 37 | +extern int dry_run; |
| 38 | extern int list_only; |
| 39 | extern int am_root; |
| 40 | extern int am_server; |
| 41 | @@ -57,6 +58,7 @@ extern int implied_dirs; |
| 42 | extern int file_extra_cnt; |
| 43 | extern int ignore_perishable; |
| 44 | extern int non_perishable_cnt; |
| 45 | +extern int checksum_updating; |
| 46 | extern int prune_empty_dirs; |
| 47 | extern int copy_links; |
| 48 | extern int copy_unsafe_links; |
| 49 | @@ -79,6 +81,9 @@ extern iconv_t ic_send, ic_recv; |
| 50 | |
| 51 | #define PTR_SIZE (sizeof (struct file_struct *)) |
| 52 | |
| 53 | +#define FLAG_SUM_MISSING (1<<1) /* F_SUM() data is undefined */ |
| 54 | +#define FLAG_SUM_KEEP (1<<2) /* keep entry when rewriting */ |
| 55 | + |
| 56 | int io_error; |
| 57 | int checksum_len; |
| 58 | dev_t filesystem_dev; /* used to implement -x */ |
| 59 | @@ -101,6 +106,10 @@ static char tmp_sum[MAX_DIGEST_LEN]; |
| 60 | static char empty_sum[MAX_DIGEST_LEN]; |
| 61 | static int flist_count_offset; /* for --delete --progress */ |
| 62 | static int dir_count = 0; |
| 63 | +static struct file_list *checksum_flist = NULL; |
| 64 | +static int checksum_matches = 0; |
| 65 | +static int checksum_updates = 0; |
| 66 | +static int regular_skipped = 0; |
| 67 | |
| 68 | static void clean_flist(struct file_list *flist, int strip_root); |
| 69 | static void output_flist(struct file_list *flist); |
| 70 | @@ -317,6 +326,304 @@ static void flist_done_allocating(struct |
| 71 | flist->pool_boundary = ptr; |
| 72 | } |
| 73 | |
| 74 | +/* The len count is the length of the basename + 1 for the null. */ |
| 75 | +static void add_checksum(const char *dirname, const char *basename, int len, |
| 76 | + OFF_T file_length, time_t mtime, time_t ctime, |
| 77 | + const char *sum, const char *alt_sum, int flags) |
| 78 | +{ |
| 79 | + struct file_struct *file; |
| 80 | + int alloc_len, extra_len; |
| 81 | + char *bp; |
| 82 | + |
| 83 | + if (len == 10+1 && *basename == '.' && strcmp(basename, ".rsyncsums") == 0) |
| 84 | + return; |
| 85 | + |
| 86 | + if (len < 0) |
| 87 | + len = strlen(basename) + 1; |
| 88 | + |
| 89 | + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) |
| 90 | + + SUM_EXTRA_CNT + TIME_EXTRA_CNT) |
| 91 | + * EXTRA_LEN; |
| 92 | +#if EXTRA_ROUNDING > 0 |
| 93 | + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) |
| 94 | + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; |
| 95 | +#endif |
| 96 | + alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1; |
| 97 | + bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); |
| 98 | + |
| 99 | + memset(bp, 0, extra_len + FILE_STRUCT_LEN); |
| 100 | + bp += extra_len; |
| 101 | + file = (struct file_struct *)bp; |
| 102 | + bp += FILE_STRUCT_LEN; |
| 103 | + |
| 104 | + memcpy(bp, basename, len); |
| 105 | + if (alt_sum) |
| 106 | + strlcpy(bp+len, alt_sum, checksum_len*2 + 1); |
| 107 | + else { |
| 108 | + memset(bp+len, '=', checksum_len*2); |
| 109 | + bp[len+checksum_len*2] = '\0'; |
| 110 | + } |
| 111 | + |
| 112 | + file->flags = flags; |
| 113 | + file->mode = S_IFREG; |
| 114 | + file->modtime = mtime; |
| 115 | + file->len32 = (uint32)file_length; |
| 116 | + if (file_length > 0xFFFFFFFFu) { |
| 117 | + file->flags |= FLAG_LENGTH64; |
| 118 | + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); |
| 119 | + } |
| 120 | + file->dirname = dirname; |
| 121 | + bp = (char*)F_SUM(file); |
| 122 | + memcpy(bp, sum, checksum_len); |
| 123 | +#if SIZEOF_TIME_T == 4 |
| 124 | + OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num = ctime; |
| 125 | +#else |
| 126 | + memcpy(bp - SIZEOF_TIME_T, &ctime, SIZEOF_TIME_T); |
| 127 | +#endif |
| 128 | + |
| 129 | + flist_expand(checksum_flist, 1); |
| 130 | + checksum_flist->files[checksum_flist->count++] = file; |
| 131 | + |
| 132 | + checksum_flist->sorted = checksum_flist->files; |
| 133 | +} |
| 134 | + |
| 135 | +static void write_checksums(const char *next_dirname, int whole_dir) |
| 136 | +{ |
| 137 | + static const char *dirname_save; |
| 138 | + char fbuf[MAXPATHLEN]; |
| 139 | + const char *dirname; |
| 140 | + int count, new_entries, counts_match, no_skipped; |
| 141 | + time_t latest_time = 0; |
| 142 | + FILE *out_fp; |
| 143 | + int i; |
| 144 | + |
| 145 | + dirname = dirname_save; |
| 146 | + dirname_save = next_dirname; |
| 147 | + |
| 148 | + if (!dirname) |
| 149 | + return; |
| 150 | + |
| 151 | + count = checksum_flist->count; |
| 152 | + new_entries = checksum_updates != 0; |
| 153 | + counts_match = count == checksum_matches; |
| 154 | + no_skipped = whole_dir && regular_skipped == 0; |
| 155 | + |
| 156 | + clean_flist(checksum_flist, 0); |
| 157 | + |
| 158 | + checksum_flist->count = 0; |
| 159 | + checksum_matches = 0; |
| 160 | + checksum_updates = 0; |
| 161 | + regular_skipped = 0; |
| 162 | + |
| 163 | + if (dry_run) |
| 164 | + return; |
| 165 | + |
| 166 | + if (*dirname) { |
| 167 | + if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf) |
| 168 | + return; |
| 169 | + } else |
| 170 | + strlcpy(fbuf, ".rsyncsums", sizeof fbuf); |
| 171 | + |
| 172 | + if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) { |
| 173 | + unlink(fbuf); |
| 174 | + return; |
| 175 | + } |
| 176 | + |
| 177 | + if (!new_entries && (counts_match || !whole_dir)) |
| 178 | + return; |
| 179 | + |
| 180 | + if (!(out_fp = fopen(fbuf, "w"))) |
| 181 | + return; |
| 182 | + |
| 183 | + new_entries = 0; |
| 184 | + for (i = checksum_flist->low; i <= checksum_flist->high; i++) { |
| 185 | + struct file_struct *file = checksum_flist->sorted[i]; |
| 186 | + const char *cp = F_SUM(file); |
| 187 | + const char *end = cp + checksum_len; |
| 188 | + const char *alt_sum = file->basename + strlen(file->basename) + 1; |
| 189 | + time_t ctime; |
| 190 | + if (whole_dir && !(file->flags & FLAG_SUM_KEEP)) |
| 191 | + continue; |
| 192 | +#if SIZEOF_TIME_T == 4 |
| 193 | + ctime = OPT_EXTRA(file, LEN64_BUMP(file) + SUM_EXTRA_CNT)->num; |
| 194 | +#else |
| 195 | + memcpy(&ctime, cp - SIZEOF_TIME_T, SIZEOF_TIME_T); |
| 196 | +#endif |
| 197 | + if (protocol_version >= 30) |
| 198 | + fprintf(out_fp, "%s ", alt_sum); |
| 199 | + if (file->flags & FLAG_SUM_MISSING) { |
| 200 | + new_entries++; |
| 201 | + do { |
| 202 | + fprintf(out_fp, "=="); |
| 203 | + } while (++cp != end); |
| 204 | + } else { |
| 205 | + do { |
| 206 | + fprintf(out_fp, "%02x", CVAL(cp, 0)); |
| 207 | + } while (++cp != end); |
| 208 | + } |
| 209 | + if (protocol_version < 30) |
| 210 | + fprintf(out_fp, " %s", alt_sum); |
| 211 | + if (*alt_sum == '=') |
| 212 | + new_entries++; |
| 213 | + fprintf(out_fp, " %10.0f %10.0f %10.0f %s\n", |
| 214 | + (double)F_LENGTH(file), (double)file->modtime, |
| 215 | + (double)ctime, file->basename); |
| 216 | + if (file->modtime > ctime) |
| 217 | + ctime = file->modtime; |
| 218 | + if (ctime > latest_time) |
| 219 | + latest_time = ctime; |
| 220 | + } |
| 221 | + |
| 222 | + fclose(out_fp); |
| 223 | + |
| 224 | + if (whole_dir && new_entries == 0) |
| 225 | + set_modtime(fbuf, latest_time, latest_time); |
| 226 | + else |
| 227 | + set_modtime(fbuf, latest_time-1, latest_time-1); |
| 228 | +} |
| 229 | + |
| 230 | +/* The direname value must remain unchanged during the lifespan of the |
| 231 | + * created checksum_flist object because we use it directly. */ |
| 232 | +static void read_checksums(const char *dirname) |
| 233 | +{ |
| 234 | + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; |
| 235 | + const char *alt_sum = NULL; |
| 236 | + OFF_T file_length; |
| 237 | + time_t mtime, ctime; |
| 238 | + int len, dlen, i, flags; |
| 239 | + char *cp; |
| 240 | + FILE *fp; |
| 241 | + |
| 242 | + write_checksums(dirname, 0); |
| 243 | + |
| 244 | + if (checksum_flist) { |
| 245 | + /* Reset the pool memory and empty the file-list array. */ |
| 246 | + pool_free_old(checksum_flist->file_pool, |
| 247 | + pool_boundary(checksum_flist->file_pool, 0)); |
| 248 | + checksum_flist->count = 0; |
| 249 | + } else |
| 250 | + checksum_flist = flist_new(FLIST_TEMP, "read_checksums"); |
| 251 | + |
| 252 | + checksum_flist->low = 0; |
| 253 | + checksum_flist->high = -1; |
| 254 | + checksum_matches = 0; |
| 255 | + checksum_updates = 0; |
| 256 | + regular_skipped = 0; |
| 257 | + |
| 258 | + if (!dirname) |
| 259 | + return; |
| 260 | + |
| 261 | + dlen = strlcpy(fbuf, dirname, sizeof fbuf); |
| 262 | + if (dlen >= (int)sizeof fbuf) |
| 263 | + return; |
| 264 | + if (dlen) |
| 265 | + fbuf[dlen++] = '/'; |
| 266 | + else |
| 267 | + dirname = NULL; |
| 268 | + strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen); |
| 269 | + if (!(fp = fopen(fbuf, "r"))) |
| 270 | + return; |
| 271 | + |
| 272 | + while (fgets(line, sizeof line, fp)) { |
| 273 | + cp = line; |
| 274 | + if (protocol_version >= 30) { |
| 275 | + alt_sum = cp; |
| 276 | + if (*cp == '=') |
| 277 | + while (*++cp == '=') {} |
| 278 | + else |
| 279 | + while (isXDigit(cp)) cp++; |
| 280 | + if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ') |
| 281 | + break; |
| 282 | + while (*++cp == ' ') {} |
| 283 | + } |
| 284 | + |
| 285 | + if (*cp == '=') { |
| 286 | + for (i = 0; i < checksum_len*2; i++, cp++) { |
| 287 | + if (*cp != '=') { |
| 288 | + cp = ""; |
| 289 | + break; |
| 290 | + } |
| 291 | + } |
| 292 | + memset(sum, 0, checksum_len); |
| 293 | + flags = FLAG_SUM_MISSING; |
| 294 | + } else { |
| 295 | + for (i = 0; i < checksum_len*2; i++, cp++) { |
| 296 | + int x; |
| 297 | + if (isXDigit(cp)) { |
| 298 | + if (isDigit(cp)) |
| 299 | + x = *cp - '0'; |
| 300 | + else |
| 301 | + x = (*cp & 0xF) + 9; |
| 302 | + } else { |
| 303 | + cp = ""; |
| 304 | + break; |
| 305 | + } |
| 306 | + if (i & 1) |
| 307 | + sum[i/2] |= x; |
| 308 | + else |
| 309 | + sum[i/2] = x << 4; |
| 310 | + } |
| 311 | + flags = 0; |
| 312 | + } |
| 313 | + if (*cp != ' ') |
| 314 | + break; |
| 315 | + while (*++cp == ' ') {} |
| 316 | + |
| 317 | + if (protocol_version < 30) { |
| 318 | + alt_sum = cp; |
| 319 | + if (*cp == '=') |
| 320 | + while (*++cp == '=') {} |
| 321 | + else |
| 322 | + while (isXDigit(cp)) cp++; |
| 323 | + if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ') |
| 324 | + break; |
| 325 | + while (*++cp == ' ') {} |
| 326 | + } |
| 327 | + |
| 328 | + file_length = 0; |
| 329 | + while (isDigit(cp)) |
| 330 | + file_length = file_length * 10 + *cp++ - '0'; |
| 331 | + if (*cp != ' ') |
| 332 | + break; |
| 333 | + while (*++cp == ' ') {} |
| 334 | + |
| 335 | + mtime = 0; |
| 336 | + while (isDigit(cp)) |
| 337 | + mtime = mtime * 10 + *cp++ - '0'; |
| 338 | + if (*cp != ' ') |
| 339 | + break; |
| 340 | + while (*++cp == ' ') {} |
| 341 | + |
| 342 | + ctime = 0; |
| 343 | + while (isDigit(cp)) |
| 344 | + ctime = ctime * 10 + *cp++ - '0'; |
| 345 | + if (*cp != ' ') |
| 346 | + break; |
| 347 | + while (*++cp == ' ') {} |
| 348 | + |
| 349 | + len = strlen(cp); |
| 350 | + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r')) |
| 351 | + len--; |
| 352 | + if (!len) |
| 353 | + break; |
| 354 | + cp[len++] = '\0'; /* len now counts the null */ |
| 355 | + if (strchr(cp, '/') || len > MAXPATHLEN) |
| 356 | + break; |
| 357 | + |
| 358 | + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); |
| 359 | + if (is_excluded(fbuf, 0, ALL_FILTERS)) { |
| 360 | + flags |= FLAG_SUM_KEEP; |
| 361 | + checksum_matches++; |
| 362 | + } |
| 363 | + |
| 364 | + add_checksum(dirname, cp, len, file_length, mtime, ctime, |
| 365 | + sum, alt_sum, flags); |
| 366 | + } |
| 367 | + fclose(fp); |
| 368 | + |
| 369 | + clean_flist(checksum_flist, 0); |
| 370 | +} |
| 371 | + |
| 372 | int push_pathname(const char *dir, int len) |
| 373 | { |
| 374 | if (dir == pathname) |
| 375 | @@ -973,34 +1280,24 @@ static struct file_struct *recv_file_ent |
| 376 | return file; |
| 377 | } |
| 378 | |
| 379 | -/** |
| 380 | - * Create a file_struct for a named file by reading its stat() |
| 381 | - * information and performing extensive checks against global |
| 382 | - * options. |
| 383 | - * |
| 384 | - * @return the new file, or NULL if there was an error or this file |
| 385 | - * should be excluded. |
| 386 | +/* Create a file_struct for a named file by reading its stat() information |
| 387 | + * and performing extensive checks against global options. |
| 388 | * |
| 389 | - * @todo There is a small optimization opportunity here to avoid |
| 390 | - * stat()ing the file in some circumstances, which has a certain cost. |
| 391 | - * We are called immediately after doing readdir(), and so we may |
| 392 | - * already know the d_type of the file. We could for example avoid |
| 393 | - * statting directories if we're not recursing, but this is not a very |
| 394 | - * important case. Some systems may not have d_type. |
| 395 | - **/ |
| 396 | + * Returns a pointer to the new file struct, or NULL if there was an error |
| 397 | + * or this file should be excluded. */ |
| 398 | struct file_struct *make_file(const char *fname, struct file_list *flist, |
| 399 | STRUCT_STAT *stp, int flags, int filter_level) |
| 400 | { |
| 401 | static char *lastdir; |
| 402 | - static int lastdir_len = -1; |
| 403 | + static int lastdir_len = -2; |
| 404 | struct file_struct *file; |
| 405 | - STRUCT_STAT st; |
| 406 | char thisname[MAXPATHLEN]; |
| 407 | char linkname[MAXPATHLEN]; |
| 408 | int alloc_len, basename_len, linkname_len; |
| 409 | int extra_len = file_extra_cnt * EXTRA_LEN; |
| 410 | const char *basename; |
| 411 | alloc_pool_t *pool; |
| 412 | + STRUCT_STAT st; |
| 413 | char *bp; |
| 414 | |
| 415 | if (strlcpy(thisname, fname, sizeof thisname) |
| 416 | @@ -1077,6 +1374,8 @@ struct file_struct *make_file(const char |
| 417 | if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { |
| 418 | if (ignore_perishable) |
| 419 | non_perishable_cnt++; |
| 420 | + if (S_ISREG(st.st_mode)) |
| 421 | + regular_skipped++; |
| 422 | return NULL; |
| 423 | } |
| 424 | |
| 425 | @@ -1115,9 +1414,16 @@ struct file_struct *make_file(const char |
| 426 | memcpy(lastdir, thisname, len); |
| 427 | lastdir[len] = '\0'; |
| 428 | lastdir_len = len; |
| 429 | + if (always_checksum && am_sender && flist) |
| 430 | + read_checksums(lastdir); |
| 431 | } |
| 432 | - } else |
| 433 | + } else { |
| 434 | basename = thisname; |
| 435 | + if (always_checksum && am_sender && flist && lastdir_len == -2) { |
| 436 | + lastdir_len = -1; |
| 437 | + read_checksums(""); |
| 438 | + } |
| 439 | + } |
| 440 | basename_len = strlen(basename) + 1; /* count the '\0' */ |
| 441 | |
| 442 | #ifdef SUPPORT_LINKS |
| 443 | @@ -1193,11 +1499,44 @@ struct file_struct *make_file(const char |
| 444 | } |
| 445 | #endif |
| 446 | |
| 447 | - if (always_checksum && am_sender && S_ISREG(st.st_mode)) |
| 448 | - file_checksum(thisname, tmp_sum, st.st_size); |
| 449 | - |
| 450 | F_PATHNAME(file) = pathname; |
| 451 | |
| 452 | + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { |
| 453 | + int j; |
| 454 | + if (flist && (j = flist_find(checksum_flist, file)) >= 0) { |
| 455 | + struct file_struct *fp = checksum_flist->sorted[j]; |
| 456 | + time_t ctime; |
| 457 | +#if SIZEOF_TIME_T == 4 |
| 458 | + ctime = OPT_EXTRA(fp, LEN64_BUMP(fp) + SUM_EXTRA_CNT)->num; |
| 459 | +#else |
| 460 | + memcpy(&ctime, F_SUM(fp) - SIZEOF_TIME_T, SIZEOF_TIME_T); |
| 461 | +#endif |
| 462 | + if (F_LENGTH(fp) == st.st_size |
| 463 | + && fp->modtime == st.st_mtime && ctime == st.st_ctime) { |
| 464 | + if (fp->flags & FLAG_SUM_MISSING) { |
| 465 | + fp->flags &= ~FLAG_SUM_MISSING; |
| 466 | + file_checksum(thisname, tmp_sum, st.st_size); |
| 467 | + memcpy((char*)F_SUM(fp), tmp_sum, MAX_DIGEST_LEN); |
| 468 | + } else { |
| 469 | + checksum_matches++; |
| 470 | + memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); |
| 471 | + } |
| 472 | + fp->flags |= FLAG_SUM_KEEP; |
| 473 | + } else { |
| 474 | + clear_file(fp); |
| 475 | + goto compute_new_checksum; |
| 476 | + } |
| 477 | + } else { |
| 478 | + compute_new_checksum: |
| 479 | + file_checksum(thisname, tmp_sum, st.st_size); |
| 480 | + if (checksum_updating && flist) { |
| 481 | + add_checksum(file->dirname, basename, basename_len, |
| 482 | + st.st_size, st.st_mtime, st.st_ctime, |
| 483 | + tmp_sum, NULL, FLAG_SUM_KEEP); |
| 484 | + } |
| 485 | + } |
| 486 | + } |
| 487 | + |
| 488 | /* This code is only used by the receiver when it is building |
| 489 | * a list of files for a delete pass. */ |
| 490 | if (keep_dirlinks && linkname_len && flist) { |
| 491 | @@ -1241,14 +1580,14 @@ void unmake_file(struct file_struct *fil |
| 492 | |
| 493 | static struct file_struct *send_file_name(int f, struct file_list *flist, |
| 494 | char *fname, STRUCT_STAT *stp, |
| 495 | - int flags, int filter_flags) |
| 496 | + int flags, int filter_level) |
| 497 | { |
| 498 | struct file_struct *file; |
| 499 | #if defined SUPPORT_ACLS || defined SUPPORT_XATTRS |
| 500 | statx sx; |
| 501 | #endif |
| 502 | |
| 503 | - file = make_file(fname, flist, stp, flags, filter_flags); |
| 504 | + file = make_file(fname, flist, stp, flags, filter_level); |
| 505 | if (!file) |
| 506 | return NULL; |
| 507 | |
| 508 | @@ -1442,7 +1781,7 @@ static void send_directory(int f, struct |
| 509 | DIR *d; |
| 510 | int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0; |
| 511 | int start = flist->count; |
| 512 | - int filter_flags = f == -2 ? SERVER_FILTERS : ALL_FILTERS; |
| 513 | + int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; |
| 514 | |
| 515 | assert(flist != NULL); |
| 516 | |
| 517 | @@ -1471,7 +1810,7 @@ static void send_directory(int f, struct |
| 518 | continue; |
| 519 | } |
| 520 | |
| 521 | - send_file_name(f, flist, fbuf, NULL, flags, filter_flags); |
| 522 | + send_file_name(f, flist, fbuf, NULL, flags, filter_level); |
| 523 | } |
| 524 | |
| 525 | fbuf[len] = '\0'; |
| 526 | @@ -1483,6 +1822,9 @@ static void send_directory(int f, struct |
| 527 | |
| 528 | closedir(d); |
| 529 | |
| 530 | + if (checksum_updating && always_checksum && am_sender && f >= 0) |
| 531 | + write_checksums(NULL, 1); |
| 532 | + |
| 533 | if (f >= 0 && recurse && !divert_dirs) { |
| 534 | int i, end = flist->count - 1; |
| 535 | /* send_if_directory() bumps flist->count, so use "end". */ |
| 536 | @@ -1916,7 +2258,11 @@ struct file_list *send_file_list(int f, |
| 537 | * file-list to check if this is a 1-file xfer. */ |
| 538 | send_extra_file_list(f, 1); |
| 539 | } |
| 540 | - } |
| 541 | + } else |
| 542 | + flist_eof = 1; |
| 543 | + |
| 544 | + if (checksum_updating && always_checksum && flist_eof) |
| 545 | + read_checksums(NULL); /* writes any last updates */ |
| 546 | |
| 547 | return flist; |
| 548 | } |
| 549 | @@ -2206,7 +2552,7 @@ void flist_free(struct file_list *flist) |
| 550 | |
| 551 | if (!flist->prev || !flist_cnt) |
| 552 | pool_destroy(flist->file_pool); |
| 553 | - else |
| 554 | + else if (flist->pool_boundary) |
| 555 | pool_free_old(flist->file_pool, flist->pool_boundary); |
| 556 | |
| 557 | if (flist->sorted && flist->sorted != flist->files) |
| 558 | @@ -2225,6 +2571,7 @@ static void clean_flist(struct file_list |
| 559 | if (!flist) |
| 560 | return; |
| 561 | if (flist->count == 0) { |
| 562 | + flist->low = 0; |
| 563 | flist->high = -1; |
| 564 | return; |
| 565 | } |
| 566 | --- old/loadparm.c |
| 567 | +++ new/loadparm.c |
| 568 | @@ -149,6 +149,7 @@ typedef struct |
| 569 | int syslog_facility; |
| 570 | int timeout; |
| 571 | |
| 572 | + BOOL checksum_updating; |
| 573 | BOOL fake_super; |
| 574 | BOOL ignore_errors; |
| 575 | BOOL ignore_nonreadable; |
| 576 | @@ -197,6 +198,7 @@ static service sDefault = |
| 577 | /* syslog_facility; */ LOG_DAEMON, |
| 578 | /* timeout; */ 0, |
| 579 | |
| 580 | + /* checksum_updating; */ False, |
| 581 | /* fake_super; */ False, |
| 582 | /* ignore_errors; */ False, |
| 583 | /* ignore_nonreadable; */ False, |
| 584 | @@ -313,6 +315,7 @@ static struct parm_struct parm_table[] = |
| 585 | {"lock file", P_STRING, P_LOCAL, &sDefault.lock_file, NULL,0}, |
| 586 | {"log file", P_STRING, P_LOCAL, &sDefault.log_file, NULL,0}, |
| 587 | {"log format", P_STRING, P_LOCAL, &sDefault.log_format, NULL,0}, |
| 588 | + {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0}, |
| 589 | {"max connections", P_INTEGER,P_LOCAL, &sDefault.max_connections, NULL,0}, |
| 590 | {"max verbosity", P_INTEGER,P_LOCAL, &sDefault.max_verbosity, NULL,0}, |
| 591 | {"name", P_STRING, P_LOCAL, &sDefault.name, NULL,0}, |
| 592 | @@ -418,6 +421,7 @@ FN_LOCAL_BOOL(lp_fake_super, fake_super) |
| 593 | FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) |
| 594 | FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) |
| 595 | FN_LOCAL_BOOL(lp_list, list) |
| 596 | +FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating) |
| 597 | FN_LOCAL_BOOL(lp_read_only, read_only) |
| 598 | FN_LOCAL_BOOL(lp_strict_modes, strict_modes) |
| 599 | FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging) |
| 600 | --- old/options.c |
| 601 | +++ new/options.c |
| 602 | @@ -109,6 +109,7 @@ size_t bwlimit_writemax = 0; |
| 603 | int ignore_existing = 0; |
| 604 | int ignore_non_existing = 0; |
| 605 | int need_messages_from_generator = 0; |
| 606 | +int checksum_updating = 0; |
| 607 | int max_delete = -1; |
| 608 | OFF_T max_size = 0; |
| 609 | OFF_T min_size = 0; |
| 610 | @@ -302,6 +303,7 @@ void usage(enum logcode F) |
| 611 | rprintf(F," -q, --quiet suppress non-error messages\n"); |
| 612 | rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); |
| 613 | rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); |
| 614 | + rprintf(F," --checksum-updating sender updates .rsyncsums files\n"); |
| 615 | rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); |
| 616 | rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); |
| 617 | rprintf(F," -r, --recursive recurse into directories\n"); |
| 618 | @@ -542,6 +544,7 @@ static struct poptOption long_options[] |
| 619 | {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, |
| 620 | {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, |
| 621 | {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, |
| 622 | + {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 }, |
| 623 | {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, |
| 624 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, |
| 625 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, |
| 626 | @@ -1896,7 +1899,9 @@ void server_options(char **args,int *arg |
| 627 | args[ac++] = basis_dir[i]; |
| 628 | } |
| 629 | } |
| 630 | - } |
| 631 | + } else if (checksum_updating) |
| 632 | + args[ac++] = "--checksum-updating"; |
| 633 | + |
| 634 | |
| 635 | if (append_mode) |
| 636 | args[ac++] = "--append"; |
| 637 | --- old/rsync.h |
| 638 | +++ new/rsync.h |
| 639 | @@ -589,6 +589,7 @@ extern int preserve_xattrs; |
| 640 | #define EXTRA_LEN (sizeof (union file_extras)) |
| 641 | #define PTR_EXTRA_LEN ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN) |
| 642 | #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN) |
| 643 | +#define TIME_EXTRA_CNT ((SIZEOF_TIME_T + EXTRA_LEN - 1) / EXTRA_LEN) |
| 644 | |
| 645 | #define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx)) |
| 646 | #define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump)) |
| 647 | @@ -1070,6 +1071,12 @@ isDigit(const char *ptr) |
| 648 | } |
| 649 | |
| 650 | static inline int |
| 651 | +isXDigit(const char *ptr) |
| 652 | +{ |
| 653 | + return isxdigit(*(unsigned char *)ptr); |
| 654 | +} |
| 655 | + |
| 656 | +static inline int |
| 657 | isPrint(const char *ptr) |
| 658 | { |
| 659 | return isprint(*(unsigned char *)ptr); |
| 660 | --- old/rsync.yo |
| 661 | +++ new/rsync.yo |
| 662 | @@ -307,6 +307,7 @@ to the detailed description below for a |
| 663 | -q, --quiet suppress non-error messages |
| 664 | --no-motd suppress daemon-mode MOTD (see caveat) |
| 665 | -c, --checksum skip based on checksum, not mod-time & size |
| 666 | + --checksum-updating sender updates .rsyncsums files |
| 667 | -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) |
| 668 | --no-OPTION turn off an implied OPTION (e.g. --no-D) |
| 669 | -r, --recursive recurse into directories |
| 670 | @@ -502,9 +503,9 @@ uses a "quick check" that (by default) c |
| 671 | of last modification match between the sender and receiver. This option |
| 672 | changes this to compare a 128-bit MD4 checksum for each file that has a |
| 673 | matching size. Generating the checksums means that both sides will expend |
| 674 | -a lot of disk I/O reading all the data in the files in the transfer (and |
| 675 | -this is prior to any reading that will be done to transfer changed files), |
| 676 | -so this can slow things down significantly. |
| 677 | +a lot of disk I/O reading the data in all the files in the transfer, so |
| 678 | +this can slow things down significantly (and this is prior to any reading |
| 679 | +that will be done to transfer the files that have changed). |
| 680 | |
| 681 | The sending side generates its checksums while it is doing the file-system |
| 682 | scan that builds the list of the available files. The receiver generates |
| 683 | @@ -512,12 +513,42 @@ its checksums when it is scanning for ch |
| 684 | file that has the same size as the corresponding sender's file: files with |
| 685 | either a changed size or a changed checksum are selected for transfer. |
| 686 | |
| 687 | +Starting with version 3.0.0, the sending side will look for a checksum |
| 688 | +summary file and use a pre-generated checksum that it reads out of the file |
| 689 | +(as long as it matches the file's size and modified time). This allows a |
| 690 | +server to support the --checksum option to clients without having to |
| 691 | +recompute the checksums for each client. See the bf(--checksum-updating) |
| 692 | +option for a way to have rsync create/update these checksum files. |
| 693 | + |
| 694 | Note that rsync always verifies that each em(transferred) file was |
| 695 | correctly reconstructed on the receiving side by checking a whole-file |
| 696 | checksum that is generated when as the file is transferred, but that |
| 697 | automatic after-the-transfer verification has nothing to do with this |
| 698 | option's before-the-transfer "Does this file need to be updated?" check. |
| 699 | |
| 700 | +dit(bf(--checksum-updating)) This option tells the sending side to create |
| 701 | +and/or update per-directory checksum files that are used by the |
| 702 | +bf(--checksum) option. The file that is updated is named .rsyncsums. If |
| 703 | +pre-transfer checksums are not being computed, this option has no effect. |
| 704 | + |
| 705 | +The checksum files stores the computed checksum, last-known size, |
| 706 | +modification time, and name for each file in the current directory. If a |
| 707 | +later transfer finds that a file matches its prior size and modification |
| 708 | +time, the checksum is assumed to still be correct. Otherwise it is |
| 709 | +recomputed and udpated in the file. |
| 710 | + |
| 711 | +To avoid transferring the system's checksum files, you can use an exclude |
| 712 | +(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use |
| 713 | +a popt alias. For instance, adding the following line in your ~/.popt file |
| 714 | +defines a bf(-cc) option that enables checksum updating and excludes the |
| 715 | +checksum files: |
| 716 | + |
| 717 | +verb( rsync alias --cc --checksum-updating --exclude=.rsyncsums) |
| 718 | + |
| 719 | +An rsync daemon does not allow the client to control this setting, so see |
| 720 | +the "checksum updating" daemon config option for information on how to make |
| 721 | +a daemon maintain these checksum files. |
| 722 | + |
| 723 | dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick |
| 724 | way of saying you want recursion and want to preserve almost |
| 725 | everything (with -H being a notable omission). |
| 726 | --- old/rsyncd.conf.yo |
| 727 | +++ new/rsyncd.conf.yo |
| 728 | @@ -198,6 +198,20 @@ locking on this file to ensure that the |
| 729 | exceeded for the modules sharing the lock file. |
| 730 | The default is tt(/var/run/rsyncd.lock). |
| 731 | |
| 732 | +dit(bf(checksum updating)) This option tells rsync to update/create the |
| 733 | +checksum information in the per-directory checksum files when users copy |
| 734 | +files using the bf(--checksum) option. Any file that has changed since it |
| 735 | +was last checksummed (or is not mentioned) has its data updated in the |
| 736 | +.rsyncsums file. |
| 737 | + |
| 738 | +Note that this updating will occur even if the module is listed as being |
| 739 | +read-only. If you want to hide these files (and you will almost always |
| 740 | +want to do), add ".rsyncsums" to the module's exclude setting. |
| 741 | + |
| 742 | +Note also that the client's command-line option, bf(--checksum-updating), |
| 743 | +has no effect on a daemon. A daemon will only update/create checksum files |
| 744 | +if this config option is true. |
| 745 | + |
| 746 | dit(bf(read only)) The "read only" option determines whether clients |
| 747 | will be able to upload files or not. If "read only" is true then any |
| 748 | attempted uploads will fail. If "read only" is false then uploads will |
| 749 | --- old/support/rsyncsums |
| 750 | +++ new/support/rsyncsums |
| 751 | @@ -0,0 +1,184 @@ |
| 752 | +#!/usr/bin/perl -w |
| 753 | +use strict; |
| 754 | + |
| 755 | +use Getopt::Long; |
| 756 | +use Cwd qw(abs_path cwd); |
| 757 | +use Digest::MD4; |
| 758 | +use Digest::MD5; |
| 759 | + |
| 760 | +our $SUMS_FILE = '.rsyncsums'; |
| 761 | + |
| 762 | +our($recurse_opt, $force_reading, $help_opt); |
| 763 | +our $verbosity = 0; |
| 764 | + |
| 765 | +&Getopt::Long::Configure('bundling'); |
| 766 | +&usage if !&GetOptions( |
| 767 | + 'recurse|r' => \$recurse_opt, |
| 768 | + 'force|f' => \$force_reading, |
| 769 | + 'verbose|v+' => \$verbosity, |
| 770 | + 'help|h' => \$help_opt, |
| 771 | +) || $help_opt; |
| 772 | + |
| 773 | +my $start_dir = cwd(); |
| 774 | + |
| 775 | +my @dirs = @ARGV; |
| 776 | +@dirs = '.' unless @dirs; |
| 777 | +foreach (@dirs) { |
| 778 | + $_ = abs_path($_); |
| 779 | +} |
| 780 | + |
| 781 | +$| = 1; |
| 782 | + |
| 783 | +my $md4 = Digest::MD4->new; |
| 784 | +my $md5 = Digest::MD5->new; |
| 785 | + |
| 786 | +while (@dirs) { |
| 787 | + my $dir = shift @dirs; |
| 788 | + |
| 789 | + if (!chdir($dir)) { |
| 790 | + warn "Unable to chdir to $dir: $!\n"; |
| 791 | + next; |
| 792 | + } |
| 793 | + if (!opendir(DP, '.')) { |
| 794 | + warn "Unable to opendir $dir: $!\n"; |
| 795 | + next; |
| 796 | + } |
| 797 | + |
| 798 | + if ($verbosity) { |
| 799 | + my $reldir = $dir; |
| 800 | + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; |
| 801 | + print "$reldir ... "; |
| 802 | + } |
| 803 | + |
| 804 | + my $sums_mtime = (stat($SUMS_FILE))[9]; |
| 805 | + my %cache; |
| 806 | + my @subdirs; |
| 807 | + my $cnt = 0; |
| 808 | + my $latest_time = 0; |
| 809 | + while (defined(my $fn = readdir(DP))) { |
| 810 | + next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn; |
| 811 | + if (-d _) { |
| 812 | + push(@subdirs, "$dir/$fn"); |
| 813 | + next; |
| 814 | + } |
| 815 | + next unless -f _; |
| 816 | + |
| 817 | + my($size,$mtime,$ctime) = (stat(_))[7,9,10]; |
| 818 | + |
| 819 | + $cache{$fn} = [ $size, $mtime, $ctime ]; |
| 820 | + $cnt++; |
| 821 | + |
| 822 | + $latest_time = $mtime if $mtime > $latest_time; |
| 823 | + $latest_time = $ctime if $ctime > $latest_time; |
| 824 | + } |
| 825 | + |
| 826 | + closedir DP; |
| 827 | + |
| 828 | + unshift(@dirs, sort @subdirs) if $recurse_opt; |
| 829 | + |
| 830 | + if (!$cnt) { |
| 831 | + if (defined $sums_mtime) { |
| 832 | + print "(removed $SUMS_FILE) " if $verbosity; |
| 833 | + unlink($SUMS_FILE); |
| 834 | + } |
| 835 | + print "empty\n" if $verbosity; |
| 836 | + next; |
| 837 | + } |
| 838 | + |
| 839 | + if (defined($sums_mtime) && $sums_mtime == $latest_time && !$force_reading) { |
| 840 | + print "OK\n" if $verbosity; |
| 841 | + next; |
| 842 | + } |
| 843 | + |
| 844 | + if (open(FP, '+<', $SUMS_FILE)) { |
| 845 | + while (<FP>) { |
| 846 | + chomp; |
| 847 | + my($sum4, $sum5, $size, $mtime, $ctime, $fn) = split(' ', $_, 6); |
| 848 | + my $ref = $cache{$fn}; |
| 849 | + if (defined $ref) { |
| 850 | + if ($$ref[0] == $size |
| 851 | + && $$ref[1] == $mtime && $$ref[2] == $ctime |
| 852 | + && $sum4 !~ /=/ && $sum5 !~ /=/) { |
| 853 | + $$ref[3] = $sum4; |
| 854 | + $$ref[4] = $sum5; |
| 855 | + $cnt--; |
| 856 | + } else { |
| 857 | + $$ref[3] = $$ref[4] = undef; |
| 858 | + } |
| 859 | + } else { |
| 860 | + $cnt = -1; # Force rewrite due to removed line. |
| 861 | + } |
| 862 | + } |
| 863 | + } else { |
| 864 | + open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n"; |
| 865 | + $cnt = -1; |
| 866 | + } |
| 867 | + |
| 868 | + if ($cnt) { |
| 869 | + print "updating\n" if $verbosity; |
| 870 | + while (my($fn, $ref) = each %cache) { |
| 871 | + next if defined $$ref[3] && defined $$ref[4]; |
| 872 | + if (!open(IN, $fn)) { |
| 873 | + print STDERR "Unable to read $fn: $!\n"; |
| 874 | + delete $cache{$fn}; |
| 875 | + next; |
| 876 | + } |
| 877 | + |
| 878 | + my($size,$mtime,$ctime) = (stat(IN))[7,9,10]; |
| 879 | + my($sum4, $sum5); |
| 880 | + |
| 881 | + while (1) { |
| 882 | + while (sysread(IN, $_, 64*1024)) { |
| 883 | + $md4->add($_); |
| 884 | + $md5->add($_); |
| 885 | + } |
| 886 | + $sum4 = $md4->hexdigest; |
| 887 | + $sum5 = $md5->hexdigest; |
| 888 | + print " $sum4 $sum5" if $verbosity > 2; |
| 889 | + print " $fn\n" if $verbosity > 1; |
| 890 | + my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10]; |
| 891 | + last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2; |
| 892 | + $size = $size2; |
| 893 | + $mtime = $mtime2; |
| 894 | + $ctime = $ctime2; |
| 895 | + sysseek(IN, 0, 0); |
| 896 | + } |
| 897 | + |
| 898 | + close IN; |
| 899 | + |
| 900 | + $cache{$fn} = [ $size, $mtime, $ctime, $sum4, $sum5 ]; |
| 901 | + } |
| 902 | + |
| 903 | + $latest_time = 0; |
| 904 | + seek(FP, 0, 0); |
| 905 | + foreach my $fn (sort keys %cache) { |
| 906 | + my $ref = $cache{$fn}; |
| 907 | + my($size, $mtime, $ctime, $sum4, $sum5) = @$ref; |
| 908 | + printf FP '%s %s %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $fn; |
| 909 | + |
| 910 | + $latest_time = $mtime if $mtime > $latest_time; |
| 911 | + $latest_time = $ctime if $ctime > $latest_time; |
| 912 | + } |
| 913 | + truncate(FP, tell(FP)); |
| 914 | + } else { |
| 915 | + print "OK.\n" if $verbosity; |
| 916 | + } |
| 917 | + |
| 918 | + close FP; |
| 919 | + |
| 920 | + utime $latest_time, $latest_time, $SUMS_FILE; |
| 921 | +} |
| 922 | + |
| 923 | +sub usage |
| 924 | +{ |
| 925 | + die <<EOT; |
| 926 | +Usage: rsyncsums [OPTIONS] [DIRS] |
| 927 | + |
| 928 | +Options: |
| 929 | + -r, --recurse Update $SUMS_FILE files in subdirectories too. |
| 930 | + -f, --force Force the reading of an $SUMS_FILE file that looks to be |
| 931 | + up-to-date. (Useful for weeding out old entries.) |
| 932 | + -v, --verbose Mention what we're doing. Repeat for more info. |
| 933 | + -h, --help Display this help message. |
| 934 | +EOT |
| 935 | +} |