1 This patch makes the processing of large really large files more efficient
2 by making sure that the sender's hash table is large enough to hold all the
3 checksum entries without being overloaded. It also makes the hashing of
4 normal sized files use slightly less memory than before.
6 An extended version of a patch by Shachar Shemesh.
10 @@ -26,63 +26,47 @@ extern int append_mode;
12 int updating_basis_file;
14 -typedef unsigned short tag;
16 -#define TABLESIZE (1<<16)
17 -#define NULL_TAG (-1)
19 static int false_alarms;
21 +static int hash_hits;
23 static int64 data_transfer;
25 static int total_false_alarms;
26 -static int total_tag_hits;
27 +static int total_hash_hits;
28 static int total_matches;
30 extern struct stats stats;
37 -static struct target *targets;
39 -static int32 *tag_table;
41 -#define gettag2(s1,s2) (((s1) + (s2)) & 0xFFFF)
42 -#define gettag(sum) gettag2((sum)&0xFFFF,(sum)>>16)
44 -static int compare_targets(struct target *t1,struct target *t2)
46 - return (int)t1->t - (int)t2->t;
48 +static uint32 tablesize;
49 +static int32 *sum_table;
51 +#define GETTAG(sum) ((sum)%tablesize)
53 static void build_hash_table(struct sum_struct *s)
56 + uint32 prior_size = tablesize;
59 - tag_table = new_array(int32, TABLESIZE);
60 + /* Dynamically calculate the hash table size so that the hash load
61 + * for big files is about 80%. This number must be odd or s2 will
62 + * not be able to span the entire set. */
63 + tablesize = (uint32)(s->count/8) * 10 + 11;
64 + if (tablesize < 65537)
65 + tablesize = 65537; /* a prime number */
66 + if (tablesize != prior_size) {
68 + sum_table = new_array(int32, tablesize);
70 + out_of_memory("build_hash_table");
73 - targets = new_array(struct target, s->count);
74 - if (!tag_table || !targets)
75 - out_of_memory("build_hash_table");
76 + memset(sum_table, 0xFF, tablesize * sizeof sum_table[0]);
78 for (i = 0; i < s->count; i++) {
80 - targets[i].t = gettag(s->sums[i].sum1);
81 + uint32 t = GETTAG(s->sums[i].sum1);
82 + s->sums[i].chain = sum_table[t];
86 - qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets);
88 - for (i = 0; i < TABLESIZE; i++)
89 - tag_table[i] = NULL_TAG;
91 - for (i = s->count; i-- > 0; )
92 - tag_table[targets[i].t] = i;
96 @@ -176,20 +160,21 @@ static void hash_search(int f,struct sum
100 - tag t = gettag2(s1,s2);
102 - int32 j = tag_table[t];
107 rprintf(FINFO,"offset=%.0f sum=%08x\n",(double)offset,sum);
116 - sum = (s1 & 0xffff) | (s2 << 16);
120 - int32 l, i = targets[j].i;
123 if (sum != s->sums[i].sum1)
125 @@ -205,9 +190,11 @@ static void hash_search(int f,struct sum
126 && !(s->sums[i].flags & SUMFLG_SAME_OFFSET))
130 - rprintf(FINFO,"potential match at %.0f target=%.0f %.0f sum=%08x\n",
131 - (double)offset,(double)j,(double)i,sum);
134 + "potential match at %.0f i=%ld sum=%08x\n",
135 + (double)offset, (long)i, sum);
139 map = (schar *)map_ptr(buf,offset,l);
140 @@ -224,8 +211,8 @@ static void hash_search(int f,struct sum
141 * one with an identical offset, so we prefer that over
142 * the following want_i optimization. */
143 if (updating_basis_file) {
145 - int32 i2 = targets[j].i;
147 + for (i2 = i; i2 >= 0; i2 = s->sums[i2].chain) {
148 if (s->sums[i2].offset != offset)
151 @@ -240,7 +227,7 @@ static void hash_search(int f,struct sum
152 * both the sender and the receiver. */
153 s->sums[i].flags |= SUMFLG_SAME_OFFSET;
155 - } while (++j < s->count && targets[j].t == t);
159 /* we've found a match, but now check to see
160 @@ -266,9 +253,9 @@ static void hash_search(int f,struct sum
164 - } while (++j < s->count && targets[j].t == t);
165 + } while ((i = s->sums[i].chain) >= 0);
169 backup = offset - last_match;
170 /* We sometimes read 1 byte prior to last_match... */
172 @@ -287,6 +274,7 @@ static void hash_search(int f,struct sum
176 + sum = (s1 & 0xffff) | (s2 << 16);
178 /* By matching early we avoid re-reading the
179 data 3 times in the case where a token
180 @@ -323,7 +311,7 @@ void match_sums(int f, struct sum_struct
189 @@ -375,16 +363,11 @@ void match_sums(int f, struct sum_struct
190 rprintf(FINFO,"sending file_sum\n");
191 write_buf(f,file_sum,MD4_SUM_LENGTH);
199 - rprintf(FINFO, "false_alarms=%d tag_hits=%d matches=%d\n",
200 - false_alarms, tag_hits, matches);
201 + rprintf(FINFO, "false_alarms=%d hash_hits=%d matches=%d\n",
202 + false_alarms, hash_hits, matches);
204 - total_tag_hits += tag_hits;
205 + total_hash_hits += hash_hits;
206 total_false_alarms += false_alarms;
207 total_matches += matches;
208 stats.literal_data += data_transfer;
209 @@ -396,8 +379,7 @@ void match_report(void)
213 - "total: matches=%d tag_hits=%d false_alarms=%d data=%.0f\n",
214 - total_matches,total_tag_hits,
215 - total_false_alarms,
216 + "total: matches=%d hash_hits=%d false_alarms=%d data=%.0f\n",
217 + total_matches, total_hash_hits, total_false_alarms,
218 (double)stats.literal_data);
222 @@ -560,6 +560,7 @@ struct sum_buf {
223 OFF_T offset; /**< offset in file of this chunk */
224 int32 len; /**< length of chunk of file */
225 uint32 sum1; /**< simple checksum */
226 + int32 chain; /**< next hash-table collision */
227 short flags; /**< flag bits */
228 char sum2[SUM_LENGTH]; /**< checksum */