1 This patch makes the processing of large really large files more efficient
2 by making sure that the sender's hash table is large enough to hold all the
3 checksum entries without being overloaded.
5 Updated to use the current hashtable method when possible, and the new
6 hashtable method (which requires a modulus calculation for up to every byte
7 of the source file) only on large files that need a larger hashtable size.
8 This avoids slowing down files that don't need the extra-large hashtable.
10 This was updated for the latest codebase from a patch written by Shachar
13 To use this patch, run these commands for a successful build:
15 patch -p1 <patches/dynamic_hash.diff
16 ./configure (optional if already run)
21 @@ -39,29 +39,50 @@ static int total_matches;
23 extern struct stats stats;
25 -#define TABLESIZE (1<<16)
26 +#define TRADITIONAL_TABLESIZE (1<<16)
28 +static uint32 tablesize;
29 static int32 *hash_table;
31 #define SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF)
32 #define SUM2HASH(sum) SUM2HASH2((sum)&0xFFFF,(sum)>>16)
34 +#define BIG_SUM2HASH(sum) ((sum)%tablesize)
36 static void build_hash_table(struct sum_struct *s)
38 + static uint32 alloc_size;
42 - hash_table = new_array(int32, TABLESIZE);
43 + /* Dynamically calculate the hash table size so that the hash load
44 + * for big files is about 80%. This number must be odd or s2 will
45 + * not be able to span the entire set. */
46 + tablesize = (uint32)(s->count/8) * 10 + 11;
47 + if (tablesize < TRADITIONAL_TABLESIZE)
48 + tablesize = TRADITIONAL_TABLESIZE;
49 + if (tablesize > alloc_size || tablesize < alloc_size - 16*1024) {
52 + hash_table = new_array(int32, tablesize);
54 out_of_memory("build_hash_table");
55 + alloc_size = tablesize;
58 - memset(hash_table, 0xFF, TABLESIZE * sizeof hash_table[0]);
59 + memset(hash_table, 0xFF, tablesize * sizeof hash_table[0]);
61 - for (i = 0; i < s->count; i++) {
62 - uint32 t = SUM2HASH(s->sums[i].sum1);
63 - s->sums[i].chain = hash_table[t];
65 + if (tablesize == TRADITIONAL_TABLESIZE) {
66 + for (i = 0; i < s->count; i++) {
67 + uint32 t = SUM2HASH(s->sums[i].sum1);
68 + s->sums[i].chain = hash_table[t];
72 + for (i = 0; i < s->count; i++) {
73 + uint32 t = BIG_SUM2HASH(s->sums[i].sum1);
74 + s->sums[i].chain = hash_table[t];
80 @@ -164,11 +185,16 @@ static void hash_search(int f,struct sum
81 (double)offset, s2 & 0xFFFF, s1 & 0xFFFF);
84 - i = hash_table[SUM2HASH2(s1,s2)];
87 + if (tablesize == TRADITIONAL_TABLESIZE) {
88 + if ((i = hash_table[SUM2HASH2(s1,s2)]) < 0)
90 + sum = (s1 & 0xffff) | (s2 << 16);
92 + sum = (s1 & 0xffff) | (s2 << 16);
93 + if ((i = hash_table[BIG_SUM2HASH(sum)]) < 0)
97 - sum = (s1 & 0xffff) | (s2 << 16);