1 This patch makes the processing of large really large files more efficient
2 by making sure that the sender's hash table is large enough to hold all the
3 checksum entries without being overloaded.
5 Updated to use the current hashtable method when possible, and the new
6 hashtable method (which requires a modulus calculation for up to every byte
7 of the source file) only on large files that need a larger hashtable size.
8 This avoids slowing down files that don't need the extra-large hashtable.
10 This was updated for the latest codebase from a patch written by Shachar
13 To use this patch, run these commands for a successful build:
15 patch -p1 <patches/dynamic_hash.diff
16 ./configure (optional if already run)
21 @@ -39,40 +39,51 @@ static int total_matches;
23 extern struct stats stats;
25 -#define TABLESIZE (1<<16)
26 +#define TRADITIONAL_TABLESIZE (1<<16)
28 +static uint32 tablesize;
29 static int32 *hash_table;
31 #define SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF)
32 #define SUM2HASH(sum) SUM2HASH2((sum)&0xFFFF,(sum)>>16)
34 -static int32 build_hash_table(struct sum_struct *s, int32 start)
35 +#define BIG_SUM2HASH(sum) ((sum)%tablesize)
37 +static void build_hash_table(struct sum_struct *s)
39 - int32 i, end = s->count;
40 + static uint32 alloc_size;
44 - hash_table = new_array(int32, TABLESIZE);
45 + /* Dynamically calculate the hash table size so that the hash load
46 + * for big files is about 80%. A number greater than the traditional
47 + * size must be odd or s2 will not be able to span the entire set. */
48 + tablesize = (uint32)(s->count/8) * 10 + 11;
49 + if (tablesize < TRADITIONAL_TABLESIZE)
50 + tablesize = TRADITIONAL_TABLESIZE;
51 + if (tablesize > alloc_size || tablesize < alloc_size - 16*1024) {
54 + hash_table = new_array(int32, tablesize);
56 out_of_memory("build_hash_table");
57 + alloc_size = tablesize;
60 - memset(hash_table, 0xFF, TABLESIZE * sizeof hash_table[0]);
62 - if (end - start > TABLESIZE*8/10)
63 - end = start + TABLESIZE*8/10;
65 - for (i = start; i < end; i++) {
66 - uint32 t = SUM2HASH(s->sums[i].sum1);
67 - s->sums[i].chain = hash_table[t];
70 + memset(hash_table, 0xFF, tablesize * sizeof hash_table[0]);
73 - rprintf(FINFO, "built hash table for entries %ld - %ld\n",
74 - (long)start, (long)end - 1);
75 + if (tablesize == TRADITIONAL_TABLESIZE) {
76 + for (i = 0; i < s->count; i++) {
77 + uint32 t = SUM2HASH(s->sums[i].sum1);
78 + s->sums[i].chain = hash_table[t];
82 + for (i = 0; i < s->count; i++) {
83 + uint32 t = BIG_SUM2HASH(s->sums[i].sum1);
84 + s->sums[i].chain = hash_table[t];
93 @@ -130,8 +141,8 @@ static void matched(int f, struct sum_st
94 static void hash_search(int f,struct sum_struct *s,
95 struct map_struct *buf, OFF_T len)
97 - OFF_T offset, end, reset = 0;
98 - int32 k, want_i, backup, sum_pos = 0;
100 + int32 k, want_i, backup;
101 char sum2[SUM_LENGTH];
104 @@ -169,24 +180,21 @@ static void hash_search(int f,struct sum
108 - if (offset >= reset) {
109 - sum_pos = build_hash_table(s, sum_pos);
110 - if (sum_pos == s->count)
113 - reset = sum_pos * s->blength;
117 rprintf(FINFO, "offset=%.0f sum=%04x%04x\n",
118 (double)offset, s2 & 0xFFFF, s1 & 0xFFFF);
121 - i = hash_table[SUM2HASH2(s1,s2)];
124 + if (tablesize == TRADITIONAL_TABLESIZE) {
125 + if ((i = hash_table[SUM2HASH2(s1,s2)]) < 0)
127 + sum = (s1 & 0xffff) | (s2 << 16);
129 + sum = (s1 & 0xffff) | (s2 << 16);
130 + if ((i = hash_table[BIG_SUM2HASH(sum)]) < 0)
134 - sum = (s1 & 0xffff) | (s2 << 16);
138 @@ -354,6 +362,11 @@ void match_sums(int f, struct sum_struct
141 if (len > 0 && s->count > 0) {
142 + build_hash_table(s);
145 + rprintf(FINFO,"built hash table\n");
147 hash_search(f, s, buf, len);