1 NOTE: this patch is for _gzip_!
3 Kevin Day's version of the gzip-rsyncable patch that uses the rsync
6 --- gzip-1.2.2/deflate.c 2004-09-15 10:28:14.000000000 -0700
7 +++ rsyncable/deflate.c 2005-02-17 14:37:14.660957200 -0700
12 +local void rsync_roll(deflate_state *s, unsigned start, unsigned num);
13 +local void rsync_roll_noop(deflate_state *s, unsigned start, unsigned num);
14 +local void rsync_roll2(deflate_state *s, unsigned start, unsigned num);
16 /* ===========================================================================
20 * See deflate.c for comments about the MIN_MATCH+1.
26 + Valid values for RSYNC_DEFAULT_CHECKSUM_TYPE are:
29 + Z_RSYNCABLE_SIMPLESUM
33 +#ifndef RSYNC_DEFAULT_CHECKSUM_TYPE
34 +# define RSYNC_DEFAULT_CHECKSUM_TYPE Z_RSYNCABLE_RSSUM
37 +#ifndef RSYNC_DEFAULT_WINDOW_SIZE
38 +# define RSYNC_DEFAULT_WINDOW_SIZE 30
41 +#ifndef RSYNC_DEFAULT_RESET_BLOCK_SIZE
42 +# define RSYNC_DEFAULT_RESET_BLOCK_SIZE 4096
45 +#ifndef RSYNC_RESET_MAGIC_VALUE
46 +# define RSYNC_RESET_MAGIC_VALUE 0
49 +#define RSYNC_SUM_MATCH(s) ((s)->rsync_sum % (s)->rsync_reset_block_size == RSYNC_RESET_MAGIC_VALUE)
50 +/* Whether window sum matches magic value */
52 +/* Global rsync mode control variable */
56 /* Values for max_lazy_match, good_match and max_chain_length, depending on
57 * the desired pack level (0..9). The values given below have been tuned to
58 * exclude worst case performance for pathological files. Better values may be
60 /* To do: ignore strm->next_in if we use it as window */
63 +int ZEXPORT deflateSetRsyncParameters_(strm, checksum_type, window_size, reset_block_size)
67 + ulg reset_block_size;
69 + deflate_state *s = strm->state;
71 + switch(checksum_type){
72 + case Z_RSYNCABLE_SIMPLESUM:
73 + s->rsync_rollfunction = rsync_roll;
75 + case Z_RSYNCABLE_RSSUM:
76 + s->rsync_rollfunction = rsync_roll2;
79 + s->rsync_rollfunction = rsync_roll_noop;
82 + s->rsync_window_size = window_size != 0 ? window_size : RSYNC_DEFAULT_WINDOW_SIZE;
83 + s->rsync_reset_block_size = reset_block_size != 0 ? reset_block_size : s->rsync_window_size;
85 + s->rsync_chunk_end = 0xFFFFFFFFUL;
93 /* ========================================================================= */
94 int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
97 s->strategy = strategy;
98 s->method = (Byte)method;
100 + deflateSetRsyncParameters_(strm, RSYNC_DEFAULT_CHECKSUM_TYPE, RSYNC_DEFAULT_WINDOW_SIZE, RSYNC_DEFAULT_RESET_BLOCK_SIZE);
102 return deflateReset(strm);
107 /* ========================================================================= */
108 int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
112 match_init(); /* initialize the asm code */
116 + s->rsync_chunk_end = 0xFFFFFFFFUL;
124 @@ -1123,6 +1201,8 @@
125 zmemcpy(s->window, s->window+wsize, (unsigned)wsize);
126 s->match_start -= wsize;
127 s->strstart -= wsize; /* we now have strstart >= MAX_DIST */
128 + if (s->rsync_chunk_end != 0xFFFFFFFFUL)
129 + s->rsync_chunk_end -= wsize;
130 s->block_start -= (long) wsize;
132 /* Slide the hash table (could be avoided with 32 bit values
133 @@ -1184,15 +1264,98 @@
134 } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
137 +local void rsync_roll(s, start, num)
144 + if (start < s->rsync_window_size) {
145 + /* before window fills. */
146 + for (i = start; i < s->rsync_window_size; i++) {
147 + if (i == start + num) return;
148 + s->rsync_sum += (ulg)s->window[i];
150 + num -= (s->rsync_window_size - start);
151 + start = s->rsync_window_size;
154 + /* buffer after window full */
155 + for (i = start; i < start+num; i++) {
156 + /* New character in */
157 + s->rsync_sum += (ulg)s->window[i];
158 + /* Old character out */
159 + s->rsync_sum -= (ulg)s->window[i - s->rsync_window_size];
160 + if (s->rsync_chunk_end == 0xFFFFFFFFUL
161 + && RSYNC_SUM_MATCH(s))
162 + s->rsync_chunk_end = i;
166 +local void rsync_roll_noop(s, start, num)
174 + Implements the 2 part rsync checksum, instead of a simple summation checksum.
176 +local void rsync_roll2(deflate_state *s, unsigned start, unsigned num)
180 + if (start < s->rsync_window_size) {
181 + /* before window fills. */
182 + for (i = start; i < s->rsync_window_size; i++) {
183 + if (i == start + num) return;
184 + s->rsync_s1 = (s->rsync_s1 + (ulg)s->window[i]) & 0xffff;
185 + s->rsync_s2 = (s->rsync_s2 + s->rsync_s1) & 0xffff;
187 + num -= (s->rsync_window_size - start);
188 + start = s->rsync_window_size;
191 + /* buffer after window full */
192 + for (i = start; i < start+num; i++) {
193 + /* Old character out */
195 + s->rsync_s1 = (s->rsync_s1 - (ulg)s->window[i - s->rsync_window_size]) & 0xffff;
196 + s->rsync_s2 = (s->rsync_s2 - s->rsync_window_size * (ulg)s->window[i - s->rsync_window_size]) & 0xffff;
198 + /* New character in */
199 + s->rsync_s1 = (s->rsync_s1 + (ulg)s->window[i]) & 0xffff;
200 + s->rsync_s2 = (s->rsync_s2 + s->rsync_s1) & 0xffff;
202 + // add the two together for the match calculation
203 + s->rsync_sum = s->rsync_s1 + s->rsync_s2;
206 + if (s->rsync_chunk_end == 0xFFFFFFFFUL
207 + && RSYNC_SUM_MATCH(s)){
208 + s->rsync_chunk_end = i;
213 +/* ===========================================================================
214 + * Set rsync_chunk_end if window sum matches magic value.
216 +#define RSYNC_ROLL(s, start, num) \
217 + do { if (zlib_rsync) (s)->rsync_rollfunction((s), (start), (num)); } while(0)
219 /* ===========================================================================
220 * Flush the current block, with given end-of-file flag.
221 * IN assertion: strstart is set to the end of the current match.
223 -#define FLUSH_BLOCK_ONLY(s, eof) { \
224 +#define FLUSH_BLOCK_ONLY(s, eof, pad) { \
225 _tr_flush_block(s, (s->block_start >= 0L ? \
226 (charf *)&s->window[(unsigned)s->block_start] : \
228 (ulg)((long)s->strstart - s->block_start), \
231 s->block_start = s->strstart; \
232 flush_pending(s->strm); \
233 @@ -1200,8 +1363,8 @@
236 /* Same but force premature exit if necessary. */
237 -#define FLUSH_BLOCK(s, eof) { \
238 - FLUSH_BLOCK_ONLY(s, eof); \
239 +#define FLUSH_BLOCK(s, eof, pad) { \
240 + FLUSH_BLOCK_ONLY(s, eof, pad); \
241 if (s->strm->avail_out == 0) return (eof) ? finish_started : need_more; \
244 @@ -1252,16 +1415,16 @@
245 /* strstart == 0 is possible when wraparound on 16-bit machine */
246 s->lookahead = (uInt)(s->strstart - max_start);
247 s->strstart = (uInt)max_start;
249 + FLUSH_BLOCK(s, 0, 0);
251 /* Flush if we may have to slide, otherwise block_start may become
252 * negative and the data will be gone:
254 if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) {
256 + FLUSH_BLOCK(s, 0, 0);
259 - FLUSH_BLOCK(s, flush == Z_FINISH);
260 + FLUSH_BLOCK(s, flush == Z_FINISH, 0);
261 return flush == Z_FINISH ? finish_done : block_done;
264 @@ -1330,6 +1493,7 @@
266 s->lookahead -= s->match_length;
268 + RSYNC_ROLL(s, s->strstart, s->match_length);
269 /* Insert new strings in the hash table only if the match length
270 * is not too large. This saves time but degrades compression.
272 @@ -1363,12 +1527,17 @@
273 /* No match, output a literal byte */
274 Tracevv((stderr,"%c", s->window[s->strstart]));
275 _tr_tally_lit (s, s->window[s->strstart], bflush);
276 + RSYNC_ROLL(s, s->strstart, 1);
280 - if (bflush) FLUSH_BLOCK(s, 0);
281 + if (zlib_rsync && s->strstart > s->rsync_chunk_end) {
282 + s->rsync_chunk_end = 0xFFFFFFFFUL;
285 + if (bflush) FLUSH_BLOCK(s, 0, bflush-1);
287 - FLUSH_BLOCK(s, flush == Z_FINISH);
288 + FLUSH_BLOCK(s, flush == Z_FINISH, bflush-1);
289 return flush == Z_FINISH ? finish_done : block_done;
292 @@ -1457,6 +1626,7 @@
294 s->lookahead -= s->prev_length-1;
296 + RSYNC_ROLL(s, s->strstart, s->prev_length+1);
298 if (++s->strstart <= max_insert) {
299 INSERT_STRING(s, s->strstart, hash_head);
300 @@ -1466,7 +1636,11 @@
301 s->match_length = MIN_MATCH-1;
304 - if (bflush) FLUSH_BLOCK(s, 0);
305 + if (zlib_rsync && s->strstart > s->rsync_chunk_end) {
306 + s->rsync_chunk_end = 0xFFFFFFFFUL;
309 + if (bflush) FLUSH_BLOCK(s, 0, bflush-1);
311 } else if (s->match_available) {
312 /* If there was no match at the previous position, output a
313 @@ -1475,9 +1649,14 @@
315 Tracevv((stderr,"%c", s->window[s->strstart-1]));
316 _tr_tally_lit(s, s->window[s->strstart-1], bflush);
317 + if (zlib_rsync && s->strstart > s->rsync_chunk_end) {
318 + s->rsync_chunk_end = 0xFFFFFFFFUL;
322 - FLUSH_BLOCK_ONLY(s, 0);
323 + FLUSH_BLOCK_ONLY(s, 0, bflush-1);
325 + RSYNC_ROLL(s, s->strstart, 1);
328 if (s->strm->avail_out == 0) return need_more;
329 @@ -1485,7 +1664,14 @@
330 /* There is no previous match to compare with, wait for
331 * the next step to decide.
333 + if (zlib_rsync && s->strstart > s->rsync_chunk_end) {
334 + /* Reset huffman tree */
335 + s->rsync_chunk_end = 0xFFFFFFFFUL;
337 + FLUSH_BLOCK(s, 0, bflush-1);
339 s->match_available = 1;
340 + RSYNC_ROLL(s, s->strstart, 1);
344 @@ -1496,7 +1682,7 @@
345 _tr_tally_lit(s, s->window[s->strstart-1], bflush);
346 s->match_available = 0;
348 - FLUSH_BLOCK(s, flush == Z_FINISH);
349 + FLUSH_BLOCK(s, flush == Z_FINISH, bflush-1);
350 return flush == Z_FINISH ? finish_done : block_done;
353 --- gzip-1.2.2/deflate.h 2004-02-24 07:38:44.000000000 -0700
354 +++ rsyncable/deflate.h 2005-02-17 13:46:12.056551200 -0700
359 + ulg rsync_sum; /* rolling sum of rsync window */
360 + ulg rsync_chunk_end; /* next rsync sequence point */
361 + ulg rsync_window_size; /* the number of bytes used in computing the rolling checksum */
362 + ulg rsync_reset_block_size; /* the compressed stream will be reset approximately every 'rsync_reset_block_size' bytes */
363 + ulg rsync_s1; /* part 1 of the checksum for use with checksum type Z_RSYNCABLE_RSSUM*/
364 + ulg rsync_s2; /* part 2 of the checksum for use with checksum type Z_RSYNCABLE_RSSUM*/
366 + /* the function that should be called for performing the rsyncable checksum roll */
367 + void (*rsync_rollfunction)(struct internal_state*s , unsigned start, unsigned num);
372 /* Output a byte on the stream.
374 void _tr_init OF((deflate_state *s));
375 int _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc));
376 void _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len,
378 + int pad, int eof));
379 void _tr_align OF((deflate_state *s));
380 void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len,
382 --- gzip-1.2.2/minigzip.c 2003-11-04 18:19:26.000000000 -0700
383 +++ rsyncable/minigzip.c 2005-02-17 13:11:35.472851600 -0700
386 gz_compress(in, out);
394 if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) {
397 - outfile[len-3] = '\0';
398 + outfile[len-3] = '.';
399 + outfile[len-2] = 'u';
400 + outfile[len-1] = 'z';
401 + outfile[len-0] = '\0';
407 gz_uncompress(in, out);
414 --- gzip-1.2.2/trees.c 2004-02-24 07:36:38.000000000 -0700
415 +++ rsyncable/trees.c 2005-02-17 13:09:38.768435100 -0700
416 @@ -918,10 +918,11 @@
417 * Determine the best encoding for the current block: dynamic trees, static
418 * trees or store, and output the encoded block to the zip file.
420 -void _tr_flush_block(s, buf, stored_len, eof)
421 +void _tr_flush_block(s, buf, stored_len, pad, eof)
423 charf *buf; /* input block, or NULL if too old */
424 ulg stored_len; /* length of input block */
425 + int pad; /* pad output to byte boundary */
426 int eof; /* true if this is the last block for a file */
428 ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
429 @@ -1009,6 +1010,12 @@
431 s->compressed_len += 7; /* align on byte boundary */
434 + } else if (pad && (s->compressed_len % 8) != 0) {
438 + _tr_stored_block(s, buf, 0, eof);
440 Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len>>3,
441 s->compressed_len-7*eof));
442 --- gzip-1.2.2/zlib.def 1969-12-31 17:00:00.000000000 -0700
443 +++ rsyncable/zlib.def 2005-02-17 14:01:48.972258000 -0700
446 +; zlib data compression library
455 +; advanced functions
456 + deflateSetDictionary
462 + inflateSetDictionary
493 +; checksum functions
496 +; various hacks, don't look :)
505 + deflateSetRsyncParameters_
506 \ No newline at end of file
507 --- gzip-1.2.2/zlib.h 2004-10-03 22:57:26.000000000 -0700
508 +++ rsyncable/zlib.h 2005-02-17 14:02:11.753362200 -0700
511 #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */
514 +/* Constants used for selecting Rsyncable checksum type */
515 +#define Z_RSYNCABLE_OFF 0
516 +#define Z_RSYNCABLE_SIMPLESUM 1
517 +#define Z_RSYNCABLE_RSSUM 2
520 #define zlib_version zlibVersion()
521 /* for compatibility with versions < 1.0.2 */
523 @@ -1185,6 +1192,17 @@
524 ZLIB_VERSION, sizeof(z_stream))
528 +/* deflateSetRsyncParameters allows for setting rsyncable parameters on a stream.
529 + These parameters MUST be set immediately after the stream is created, and before
530 + any data is written to the stream.
532 +ZEXTERN int ZEXPORT deflateSetRsyncParameters_ OF((z_stream FAR *strm, int checksum_type, unsigned long window_size, unsigned long reset_block_size));
534 +#define deflateSetRsyncParameters(strm, checksum_type, window_size, reset_block_size) \
535 + deflateSetRsyncParameters_((strm), (checksum_type), (window_size), (reset_block_size))
538 #if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
539 struct internal_state {int dummy;}; /* hack for buggy compilers */
541 @@ -1193,6 +1211,10 @@
542 ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z));
543 ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void));
545 +/* Global rsync mode control variable */
546 +extern int zlib_rsync;