1 This patch makes the processing of large really large files more efficient
2 by making sure that the sender's hash table is large enough to hold all the
3 checksum entries without being overloaded. It also makes the hashing of
4 normal sized files use slightly less memory than before.
6 An extended version of a patch by Shachar Shemesh.
10 @@ -26,11 +26,6 @@ extern int append_mode;
12 int updating_basis_file;
14 -typedef unsigned short tag;
16 -#define TABLESIZE (1<<16)
17 -#define NULL_TAG (-1)
19 static int false_alarms;
22 @@ -42,47 +37,36 @@ static int total_matches;
24 extern struct stats stats;
31 -static struct target *targets;
33 -static int32 *tag_table;
35 -#define gettag2(s1,s2) (((s1) + (s2)) & 0xFFFF)
36 -#define gettag(sum) gettag2((sum)&0xFFFF,(sum)>>16)
38 -static int compare_targets(struct target *t1,struct target *t2)
40 - return (int)t1->t - (int)t2->t;
42 +static int32 tablesize;
43 +static int32 *sum_table;
45 +#define gettag2(s1,s2) gettag((s1) + ((s2)<<16))
46 +#define gettag(sum) ((sum)%tablesize)
48 static void build_hash_table(struct sum_struct *s)
51 + int32 i, prior_size = tablesize;
54 - tag_table = new_array(int32, TABLESIZE);
55 + /* Dynamically calculate the hash table size so that the hash load
56 + * for big files is about 80%. This number must be odd or s2 will
57 + * not be able to span the entire set. */
58 + tablesize = (s->count/8) * 10 + 11;
59 + if (tablesize < 65537)
60 + tablesize = 65537; /* a prime number */
61 + if (tablesize != prior_size) {
63 + sum_table = new_array(int32, tablesize);
65 + out_of_memory("build_hash_table");
68 - targets = new_array(struct target, s->count);
69 - if (!tag_table || !targets)
70 - out_of_memory("build_hash_table");
71 + memset(sum_table, 0xFF, tablesize * sizeof (sum_table[0]));
73 for (i = 0; i < s->count; i++) {
75 - targets[i].t = gettag(s->sums[i].sum1);
76 + int32 t = gettag(s->sums[i].sum1);
77 + s->sums[i].chain = sum_table[t];
81 - qsort(targets,s->count,sizeof(targets[0]),(int (*)())compare_targets);
83 - for (i = 0; i < TABLESIZE; i++)
84 - tag_table[i] = NULL_TAG;
86 - for (i = s->count; i-- > 0; )
87 - tag_table[targets[i].t] = i;
91 @@ -176,20 +160,16 @@ static void hash_search(int f,struct sum
95 - tag t = gettag2(s1,s2);
96 + int32 i, t = gettag2(s1,s2);
98 - int32 j = tag_table[t];
101 rprintf(FINFO,"offset=%.0f sum=%08x\n",(double)offset,sum);
106 sum = (s1 & 0xffff) | (s2 << 16);
109 - int32 l, i = targets[j].i;
110 + for (i = sum_table[t]; i >= 0; i = s->sums[i].chain) {
113 if (sum != s->sums[i].sum1)
115 @@ -205,9 +185,10 @@ static void hash_search(int f,struct sum
116 && !(s->sums[i].flags & SUMFLG_SAME_OFFSET))
120 - rprintf(FINFO,"potential match at %.0f target=%.0f %.0f sum=%08x\n",
121 - (double)offset,(double)j,(double)i,sum);
123 + rprintf(FINFO,"potential match at %.0f i=%ld sum=%08x\n",
124 + (double)offset, (long)i, sum);
128 map = (schar *)map_ptr(buf,offset,l);
129 @@ -224,23 +205,23 @@ static void hash_search(int f,struct sum
130 * one with an identical offset, so we prefer that over
131 * the following want_i optimization. */
132 if (updating_basis_file) {
134 - int32 i2 = targets[j].i;
136 + for (i2 = i; i2 >= 0; i2 = s->sums[i2].chain) {
137 if (s->sums[i2].offset != offset)
140 if (sum != s->sums[i2].sum1)
143 if (memcmp(sum2, s->sums[i2].sum2,
149 /* This chunk was at the same offset on
150 * both the sender and the receiver. */
151 s->sums[i].flags |= SUMFLG_SAME_OFFSET;
153 - } while (++j < s->count && targets[j].t == t);
157 /* we've found a match, but now check to see
158 @@ -266,9 +247,8 @@ static void hash_search(int f,struct sum
162 - } while (++j < s->count && targets[j].t == t);
166 backup = offset - last_match;
167 /* We sometimes read 1 byte prior to last_match... */
169 @@ -375,11 +355,6 @@ void match_sums(int f, struct sum_struct
170 rprintf(FINFO,"sending file_sum\n");
171 write_buf(f,file_sum,MD4_SUM_LENGTH);
179 rprintf(FINFO, "false_alarms=%d tag_hits=%d matches=%d\n",
180 false_alarms, tag_hits, matches);
183 @@ -560,6 +560,7 @@ struct sum_buf {
184 OFF_T offset; /**< offset in file of this chunk */
185 int32 len; /**< length of chunk of file */
186 uint32 sum1; /**< simple checksum */
187 + int32 chain; /**< next hash-table collision */
188 short flags; /**< flag bits */
189 char sum2[SUM_LENGTH]; /**< checksum */
193 @@ -92,6 +92,7 @@ static struct sum_struct *receive_sums(i
195 s->sums[i].offset = offset;
196 s->sums[i].flags = 0;
197 + s->sums[i].chain = -1;
199 if (i == s->count-1 && s->remainder != 0)
200 s->sums[i].len = s->remainder;