X-Git-Url: https://mattmccutchen.net/rsync/rsync-patches.git/blobdiff_plain/1db27b7c1a718380ee162ee30bf3b4707d65e851..9c25eef5a53f1970a5379ac5320ff4172d08959a:/dynamic_hash.diff diff --git a/dynamic_hash.diff b/dynamic_hash.diff index 480384d..f9a41d5 100644 --- a/dynamic_hash.diff +++ b/dynamic_hash.diff @@ -1,62 +1,41 @@ This patch makes the processing of large really large files more efficient by making sure that the sender's hash table is large enough to hold all the -checksum entries without being overloaded. It also makes the hashing of -normal sized files use slightly less memory than before. +checksum entries without being overloaded. Unfortunately, the code adds a +modulus calculation for (up to) every byte of the source file, which slows +down the code for normal file sizes (e.g. 4 CPU seconds slower on a Pentium +III when copying a 65 MB file without very much matching data). -An extended version of a patch by Shachar Shemesh. +This was updated for the latest codebase from a patch written by Shachar +Shemesh. + +To use this patch, run these commands for a successful build: + + patch -p1 >16) +-#define TABLESIZE (1<<16) - --static int compare_targets(struct target *t1,struct target *t2) --{ -- return (int)t1->t - (int)t2->t; --} +static uint32 tablesize; -+static int32 *sum_table; + static int32 *hash_table; -+#define GETTAG(sum) ((sum)%tablesize) +-#define SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF) +-#define SUM2HASH(sum) SUM2HASH2((sum)&0xFFFF,(sum)>>16) ++#define SUM2HASH(sum) ((sum)%tablesize) static void build_hash_table(struct sum_struct *s) { int32 i; + uint32 prior_size = tablesize; -- if (!tag_table) -- tag_table = new_array(int32, TABLESIZE); +- if (!hash_table) { +- hash_table = new_array(int32, TABLESIZE); + /* Dynamically calculate the hash table size so that the hash load + * for big files is about 80%. This number must be odd or s2 will + * not be able to span the entire set. */ @@ -64,166 +43,29 @@ An extended version of a patch by Shachar Shemesh. + if (tablesize < 65537) + tablesize = 65537; /* a prime number */ + if (tablesize != prior_size) { -+ free(sum_table); -+ sum_table = new_array(int32, tablesize); -+ if (!sum_table) -+ out_of_memory("build_hash_table"); -+ } - -- targets = new_array(struct target, s->count); -- if (!tag_table || !targets) -- out_of_memory("build_hash_table"); -+ memset(sum_table, 0xFF, tablesize * sizeof sum_table[0]); - - for (i = 0; i < s->count; i++) { -- targets[i].i = i; -- targets[i].t = gettag(s->sums[i].sum1); -+ uint32 t = GETTAG(s->sums[i].sum1); -+ s->sums[i].chain = sum_table[t]; -+ sum_table[t] = i; - } -- -- qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets); -- -- for (i = 0; i < TABLESIZE; i++) -- tag_table[i] = NULL_TAG; -- -- for (i = s->count; i-- > 0; ) -- tag_table[targets[i].t] = i; - } - - -@@ -176,20 +160,21 @@ static void hash_search(int f,struct sum ++ if (hash_table) ++ free(hash_table); ++ hash_table = new_array(int32, tablesize); + if (!hash_table) + out_of_memory("build_hash_table"); } - do { -- tag t = gettag2(s1,s2); - int done_csum2 = 0; -- int32 j = tag_table[t]; -+ int32 i; -+ uint32 t; +- memset(hash_table, 0xFF, TABLESIZE * sizeof hash_table[0]); ++ memset(hash_table, 0xFF, tablesize * sizeof hash_table[0]); - if (verbose > 4) - rprintf(FINFO,"offset=%.0f sum=%08x\n",(double)offset,sum); + for (i = 0; i < s->count; i++) { + uint32 t = SUM2HASH(s->sums[i].sum1); +@@ -165,11 +172,11 @@ static void hash_search(int f,struct sum + (double)offset, s2 & 0xFFFF, s1 & 0xFFFF); + } -- if (j == NULL_TAG) -- goto null_tag; -+ t = GETTAG(sum); -+ i = sum_table[t]; -+ if (i < 0) -+ goto null_hash; +- i = hash_table[SUM2HASH2(s1,s2)]; ++ sum = (s1 & 0xffff) | (s2 << 16); ++ i = hash_table[SUM2HASH(sum)]; + if (i < 0) + goto null_hash; - sum = (s1 & 0xffff) | (s2 << 16); -- tag_hits++; -+ hash_hits++; + hash_hits++; do { -- int32 l, i = targets[j].i; -+ int32 l; - - if (sum != s->sums[i].sum1) - continue; -@@ -205,9 +190,11 @@ static void hash_search(int f,struct sum - && !(s->sums[i].flags & SUMFLG_SAME_OFFSET)) - continue; - -- if (verbose > 3) -- rprintf(FINFO,"potential match at %.0f target=%.0f %.0f sum=%08x\n", -- (double)offset,(double)j,(double)i,sum); -+ if (verbose > 3) { -+ rprintf(FINFO, -+ "potential match at %.0f i=%ld sum=%08x\n", -+ (double)offset, (long)i, sum); -+ } - - if (!done_csum2) { - map = (schar *)map_ptr(buf,offset,l); -@@ -224,8 +211,8 @@ static void hash_search(int f,struct sum - * one with an identical offset, so we prefer that over - * the following want_i optimization. */ - if (updating_basis_file) { -- do { -- int32 i2 = targets[j].i; -+ int32 i2; -+ for (i2 = i; i2 >= 0; i2 = s->sums[i2].chain) { - if (s->sums[i2].offset != offset) - continue; - if (i2 != i) { -@@ -240,7 +227,7 @@ static void hash_search(int f,struct sum - * both the sender and the receiver. */ - s->sums[i].flags |= SUMFLG_SAME_OFFSET; - goto set_want_i; -- } while (++j < s->count && targets[j].t == t); -+ } - } - - /* we've found a match, but now check to see -@@ -266,9 +253,9 @@ static void hash_search(int f,struct sum - s2 = sum >> 16; - matches++; - break; -- } while (++j < s->count && targets[j].t == t); -+ } while ((i = s->sums[i].chain) >= 0); - -- null_tag: -+ null_hash: - backup = offset - last_match; - /* We sometimes read 1 byte prior to last_match... */ - if (backup < 0) -@@ -287,6 +274,7 @@ static void hash_search(int f,struct sum - s2 += s1; - } else - --k; -+ sum = (s1 & 0xffff) | (s2 << 16); - - /* By matching early we avoid re-reading the - data 3 times in the case where a token -@@ -323,7 +311,7 @@ void match_sums(int f, struct sum_struct - - last_match = 0; - false_alarms = 0; -- tag_hits = 0; -+ hash_hits = 0; - matches = 0; - data_transfer = 0; - -@@ -375,16 +363,11 @@ void match_sums(int f, struct sum_struct - rprintf(FINFO,"sending file_sum\n"); - write_buf(f,file_sum,MD4_SUM_LENGTH); - -- if (targets) { -- free(targets); -- targets=NULL; -- } -- - if (verbose > 2) -- rprintf(FINFO, "false_alarms=%d tag_hits=%d matches=%d\n", -- false_alarms, tag_hits, matches); -+ rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n", -+ false_alarms, hash_hits, matches); - -- total_tag_hits += tag_hits; -+ total_hash_hits += hash_hits; - total_false_alarms += false_alarms; - total_matches += matches; - stats.literal_data += data_transfer; -@@ -396,8 +379,7 @@ void match_report(void) - return; - - rprintf(FINFO, -- "total: matches=%d tag_hits=%d false_alarms=%d data=%.0f\n", -- total_matches,total_tag_hits, -- total_false_alarms, -+ "total: matches=%d hash_hits=%d false_alarms=%d data=%.0f\n", -+ total_matches, total_hash_hits, total_false_alarms, - (double)stats.literal_data); - } ---- old/rsync.h -+++ new/rsync.h -@@ -560,6 +560,7 @@ struct sum_buf { - OFF_T offset; /**< offset in file of this chunk */ - int32 len; /**< length of chunk of file */ - uint32 sum1; /**< simple checksum */ -+ int32 chain; /**< next hash-table collision */ - short flags; /**< flag bits */ - char sum2[SUM_LENGTH]; /**< checksum */ - }; + int32 l;