X-Git-Url: https://mattmccutchen.net/rsync/rsync-patches.git/blobdiff_plain/c541912faf23e7b364ce23f2e36054b5f86417bc..d705d4dd33c2cd5a83d772df98c7b3ff5319af34:/dynamic_hash.diff diff --git a/dynamic_hash.diff b/dynamic_hash.diff index cd002e9..f9a41d5 100644 --- a/dynamic_hash.diff +++ b/dynamic_hash.diff @@ -1,57 +1,41 @@ This patch makes the processing of large really large files more efficient by making sure that the sender's hash table is large enough to hold all the -checksum entries without being overloaded. It also makes the hashing of -normal sized files use slightly less memory than before. +checksum entries without being overloaded. Unfortunately, the code adds a +modulus calculation for (up to) every byte of the source file, which slows +down the code for normal file sizes (e.g. 4 CPU seconds slower on a Pentium +III when copying a 65 MB file without very much matching data). -An extended version of a patch by Shachar Shemesh. +This was updated for the latest codebase from a patch written by Shachar +Shemesh. + +To use this patch, run these commands for a successful build: + + patch -p1 >16) +-#define TABLESIZE (1<<16) - --static int compare_targets(struct target *t1,struct target *t2) --{ -- return (int)t1->t - (int)t2->t; --} +static uint32 tablesize; -+static int32 *sum_table; + static int32 *hash_table; -+#define gettag2(s1,s2) gettag((s1) + ((s2)<<16)) -+#define gettag(sum) ((sum)%tablesize) +-#define SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF) +-#define SUM2HASH(sum) SUM2HASH2((sum)&0xFFFF,(sum)>>16) ++#define SUM2HASH(sum) ((sum)%tablesize) static void build_hash_table(struct sum_struct *s) { int32 i; + uint32 prior_size = tablesize; -- if (!tag_table) -- tag_table = new_array(int32, TABLESIZE); +- if (!hash_table) { +- hash_table = new_array(int32, TABLESIZE); + /* Dynamically calculate the hash table size so that the hash load + * for big files is about 80%. This number must be odd or s2 will + * not be able to span the entire set. */ @@ -59,143 +43,29 @@ An extended version of a patch by Shachar Shemesh. + if (tablesize < 65537) + tablesize = 65537; /* a prime number */ + if (tablesize != prior_size) { -+ free(sum_table); -+ sum_table = new_array(int32, tablesize); -+ if (!sum_table) -+ out_of_memory("build_hash_table"); -+ } - -- targets = new_array(struct target, s->count); -- if (!tag_table || !targets) -- out_of_memory("build_hash_table"); -+ memset(sum_table, 0xFF, tablesize * sizeof (sum_table[0])); - - for (i = 0; i < s->count; i++) { -- targets[i].i = i; -- targets[i].t = gettag(s->sums[i].sum1); -+ uint32 t = gettag(s->sums[i].sum1); -+ s->sums[i].chain = sum_table[t]; -+ sum_table[t] = i; - } -- -- qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets); -- -- for (i = 0; i < TABLESIZE; i++) -- tag_table[i] = NULL_TAG; -- -- for (i = s->count; i-- > 0; ) -- tag_table[targets[i].t] = i; - } - - -@@ -176,20 +161,17 @@ static void hash_search(int f,struct sum ++ if (hash_table) ++ free(hash_table); ++ hash_table = new_array(int32, tablesize); + if (!hash_table) + out_of_memory("build_hash_table"); } - do { -- tag t = gettag2(s1,s2); -+ uint32 t = gettag2(s1,s2); - int done_csum2 = 0; -- int32 j = tag_table[t]; -+ int32 i; - - if (verbose > 4) - rprintf(FINFO,"offset=%.0f sum=%08x\n",(double)offset,sum); - -- if (j == NULL_TAG) -- goto null_tag; -- - sum = (s1 & 0xffff) | (s2 << 16); - tag_hits++; -- do { -- int32 l, i = targets[j].i; -+ for (i = sum_table[t]; i >= 0; i = s->sums[i].chain) { -+ int32 l; - - if (sum != s->sums[i].sum1) - continue; -@@ -205,9 +187,10 @@ static void hash_search(int f,struct sum - && !(s->sums[i].flags & SUMFLG_SAME_OFFSET)) - continue; +- memset(hash_table, 0xFF, TABLESIZE * sizeof hash_table[0]); ++ memset(hash_table, 0xFF, tablesize * sizeof hash_table[0]); -- if (verbose > 3) -- rprintf(FINFO,"potential match at %.0f target=%.0f %.0f sum=%08x\n", -- (double)offset,(double)j,(double)i,sum); -+ if (verbose > 3) { -+ rprintf(FINFO,"potential match at %.0f i=%ld sum=%08x\n", -+ (double)offset, (long)i, sum); -+ } - - if (!done_csum2) { - map = (schar *)map_ptr(buf,offset,l); -@@ -224,23 +207,23 @@ static void hash_search(int f,struct sum - * one with an identical offset, so we prefer that over - * the following want_i optimization. */ - if (updating_basis_file) { -- do { -- int32 i2 = targets[j].i; -+ int32 i2; -+ for (i2 = i; i2 >= 0; i2 = s->sums[i2].chain) { - if (s->sums[i2].offset != offset) - continue; - if (i2 != i) { - if (sum != s->sums[i2].sum1) -- break; -+ continue; - if (memcmp(sum2, s->sums[i2].sum2, - s->s2length) != 0) -- break; -+ continue; - i = i2; - } - /* This chunk was at the same offset on - * both the sender and the receiver. */ - s->sums[i].flags |= SUMFLG_SAME_OFFSET; - goto set_want_i; -- } while (++j < s->count && targets[j].t == t); -+ } - } - - /* we've found a match, but now check to see -@@ -266,9 +249,8 @@ static void hash_search(int f,struct sum - s2 = sum >> 16; - matches++; - break; -- } while (++j < s->count && targets[j].t == t); -+ } - -- null_tag: - backup = offset - last_match; - /* We sometimes read 1 byte prior to last_match... */ - if (backup < 0) -@@ -375,11 +357,6 @@ void match_sums(int f, struct sum_struct - rprintf(FINFO,"sending file_sum\n"); - write_buf(f,file_sum,MD4_SUM_LENGTH); - -- if (targets) { -- free(targets); -- targets=NULL; -- } -- - if (verbose > 2) - rprintf(FINFO, "false_alarms=%d tag_hits=%d matches=%d\n", - false_alarms, tag_hits, matches); ---- old/rsync.h -+++ new/rsync.h -@@ -560,6 +560,7 @@ struct sum_buf { - OFF_T offset; /**< offset in file of this chunk */ - int32 len; /**< length of chunk of file */ - uint32 sum1; /**< simple checksum */ -+ int32 chain; /**< next hash-table collision */ - short flags; /**< flag bits */ - char sum2[SUM_LENGTH]; /**< checksum */ - }; ---- old/sender.c -+++ new/sender.c -@@ -92,6 +92,7 @@ static struct sum_struct *receive_sums(i - - s->sums[i].offset = offset; - s->sums[i].flags = 0; -+ s->sums[i].chain = -1; - - if (i == s->count-1 && s->remainder != 0) - s->sums[i].len = s->remainder; + for (i = 0; i < s->count; i++) { + uint32 t = SUM2HASH(s->sums[i].sum1); +@@ -165,11 +172,11 @@ static void hash_search(int f,struct sum + (double)offset, s2 & 0xFFFF, s1 & 0xFFFF); + } + +- i = hash_table[SUM2HASH2(s1,s2)]; ++ sum = (s1 & 0xffff) | (s2 << 16); ++ i = hash_table[SUM2HASH(sum)]; + if (i < 0) + goto null_hash; + +- sum = (s1 & 0xffff) | (s2 << 16); + hash_hits++; + do { + int32 l;