token.c

   1 /*
   2  * Routines used by the file-transfer code.
   3  *
   4  * Copyright (C) 1996 Andrew Tridgell
   5  * Copyright (C) 1996 Paul Mackerras
   6  * Copyright (C) 2003-2007 Wayne Davison
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 3 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, visit the http://fsf.org website.
  19  */
  20
  21 #include "rsync.h"
  22 #include "zlib/zlib.h"
  23
  24 extern int do_compression;
  25 extern int module_id;
  26 extern int def_compress_level;
  27
  28 static int compression_level, per_file_default_level;
  29
  30 /* determine the compression level based on a wildcard filename list */
  31 void set_compression(char *fname)
  32 {
  33         static char *match_list;
  34         char *s;
  35
  36         if (!do_compression)
  37                 return;
  38
  39         if (!match_list) {
  40                 char *t, *f = lp_dont_compress(module_id);
  41                 int len = strlen(f);
  42                 if (!(match_list = t = new_array(char, len + 2)))
  43                         out_of_memory("set_compression");
  44                 while (*f) {
  45                         if (*f == ' ') {
  46                                 f++;
  47                                 continue;
  48                         }
  49                         do {
  50                                 if (isUpper(f))
  51                                         *t++ = toLower(f);
  52                                 else
  53                                         *t++ = *f;
  54                         } while (*++f != ' ' && *f);
  55                         *t++ = '\0';
  56                 }
  57                 /* Optimize a match-string of "*". */
  58                 if (t - match_list == 2 && match_list[0] == '*') {
  59                         t = match_list;
  60                         per_file_default_level = 0;
  61                 } else
  62                         per_file_default_level = def_compress_level;
  63                 *t++ = '\0';
  64         }
  65
  66         compression_level = per_file_default_level;
  67
  68         if (!*match_list)
  69                 return;
  70
  71         if ((s = strrchr(fname, '/')) != NULL)
  72                 fname = s + 1;
  73
  74         for (s = match_list; *s; s += strlen(s) + 1) {
  75                 if (iwildmatch(s, fname)) {
  76                         compression_level = 0;
  77                         break;
  78                 }
  79         }
  80 }
  81
  82 /* non-compressing recv token */
  83 static int32 simple_recv_token(int f, char **data)
  84 {
  85         static int32 residue;
  86         static char *buf;
  87         int32 n;
  88
  89         if (!buf) {
  90                 buf = new_array(char, CHUNK_SIZE);
  91                 if (!buf)
  92                         out_of_memory("simple_recv_token");
  93         }
  94
  95         if (residue == 0) {
  96                 int32 i = read_int(f);
  97                 if (i <= 0)
  98                         return i;
  99                 residue = i;
 100         }
 101
 102         *data = buf;
 103         n = MIN(CHUNK_SIZE,residue);
 104         residue -= n;
 105         read_buf(f,buf,n);
 106         return n;
 107 }
 108
 109 /* non-compressing send token */
 110 static void simple_send_token(int f, int32 token, struct map_struct *buf,
 111                               OFF_T offset, int32 n)
 112 {
 113         if (n > 0) {
 114                 int32 len = 0;
 115                 while (len < n) {
 116                         int32 n1 = MIN(CHUNK_SIZE, n-len);
 117                         write_int(f, n1);
 118                         write_buf(f, map_ptr(buf, offset+len, n1), n1);
 119                         len += n1;
 120                 }
 121         }
 122         /* a -2 token means to send data only and no token */
 123         if (token != -2)
 124                 write_int(f, -(token+1));
 125 }
 126
 127 /* Flag bytes in compressed stream are encoded as follows: */
 128 #define END_FLAG        0       /* that's all folks */
 129 #define TOKEN_LONG      0x20    /* followed by 32-bit token number */
 130 #define TOKENRUN_LONG   0x21    /* ditto with 16-bit run count */
 131 #define DEFLATED_DATA   0x40    /* + 6-bit high len, then low len byte */
 132 #define TOKEN_REL       0x80    /* + 6-bit relative token number */
 133 #define TOKENRUN_REL    0xc0    /* ditto with 16-bit run count */
 134
 135 #define MAX_DATA_COUNT  16383   /* fit 14 bit count into 2 bytes with flags */
 136
 137 /* zlib.h says that if we want to be able to compress something in a single
 138  * call, avail_out must be at least 0.1% larger than avail_in plus 12 bytes.
 139  * We'll add in 0.1%+16, just to be safe (and we'll avoid floating point,
 140  * to ensure that this is a compile-time value). */
 141 #define AVAIL_OUT_SIZE(avail_in_size) ((avail_in_size)*1001/1000+16)
 142
 143 /* For coding runs of tokens */
 144 static int32 last_token = -1;
 145 static int32 run_start;
 146 static int32 last_run_end;
 147
 148 /* Deflation state */
 149 static z_stream tx_strm;
 150
 151 /* Output buffer */
 152 static char *obuf;
 153
 154 /* We want obuf to be able to hold both MAX_DATA_COUNT+2 bytes as well as
 155  * AVAIL_OUT_SIZE(CHUNK_SIZE) bytes, so make sure that it's large enough. */
 156 #if MAX_DATA_COUNT+2 > AVAIL_OUT_SIZE(CHUNK_SIZE)
 157 #define OBUF_SIZE       (MAX_DATA_COUNT+2)
 158 #else
 159 #define OBUF_SIZE       AVAIL_OUT_SIZE(CHUNK_SIZE)
 160 #endif
 161
 162 /* Send a deflated token */
 163 static void
 164 send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 165                     int32 nb, int32 toklen)
 166 {
 167         int32 n, r;
 168         static int init_done, flush_pending;
 169
 170         if (last_token == -1) {
 171                 /* initialization */
 172                 if (!init_done) {
 173                         tx_strm.next_in = NULL;
 174                         tx_strm.zalloc = NULL;
 175                         tx_strm.zfree = NULL;
 176                         if (deflateInit2(&tx_strm, compression_level,
 177                                          Z_DEFLATED, -15, 8,
 178                                          Z_DEFAULT_STRATEGY) != Z_OK) {
 179                                 rprintf(FERROR, "compression init failed\n");
 180                                 exit_cleanup(RERR_STREAMIO);
 181                         }
 182                         if ((obuf = new_array(char, OBUF_SIZE)) == NULL)
 183                                 out_of_memory("send_deflated_token");
 184                         init_done = 1;
 185                 } else
 186                         deflateReset(&tx_strm);
 187                 last_run_end = 0;
 188                 run_start = token;
 189                 flush_pending = 0;
 190         } else if (last_token == -2) {
 191                 run_start = token;
 192         } else if (nb != 0 || token != last_token + 1
 193                    || token >= run_start + 65536) {
 194                 /* output previous run */
 195                 r = run_start - last_run_end;
 196                 n = last_token - run_start;
 197                 if (r >= 0 && r <= 63) {
 198                         write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
 199                 } else {
 200                         write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
 201                         write_int(f, run_start);
 202                 }
 203                 if (n != 0) {
 204                         write_byte(f, n);
 205                         write_byte(f, n >> 8);
 206                 }
 207                 last_run_end = last_token;
 208                 run_start = token;
 209         }
 210
 211         last_token = token;
 212
 213         if (nb != 0 || flush_pending) {
 214                 /* deflate the data starting at offset */
 215                 int flush = Z_NO_FLUSH;
 216                 tx_strm.avail_in = 0;
 217                 tx_strm.avail_out = 0;
 218                 do {
 219                         if (tx_strm.avail_in == 0 && nb != 0) {
 220                                 /* give it some more input */
 221                                 n = MIN(nb, CHUNK_SIZE);
 222                                 tx_strm.next_in = (Bytef *)
 223                                         map_ptr(buf, offset, n);
 224                                 tx_strm.avail_in = n;
 225                                 nb -= n;
 226                                 offset += n;
 227                         }
 228                         if (tx_strm.avail_out == 0) {
 229                                 tx_strm.next_out = (Bytef *)(obuf + 2);
 230                                 tx_strm.avail_out = MAX_DATA_COUNT;
 231                                 if (flush != Z_NO_FLUSH) {
 232                                         /*
 233                                          * We left the last 4 bytes in the
 234                                          * buffer, in case they are the
 235                                          * last 4.  Move them to the front.
 236                                          */
 237                                         memcpy(tx_strm.next_out,
 238                                                obuf+MAX_DATA_COUNT-2, 4);
 239                                         tx_strm.next_out += 4;
 240                                         tx_strm.avail_out -= 4;
 241                                 }
 242                         }
 243                         if (nb == 0 && token != -2)
 244                                 flush = Z_SYNC_FLUSH;
 245                         r = deflate(&tx_strm, flush);
 246                         if (r != Z_OK) {
 247                                 rprintf(FERROR, "deflate returned %d\n", r);
 248                                 exit_cleanup(RERR_STREAMIO);
 249                         }
 250                         if (nb == 0 || tx_strm.avail_out == 0) {
 251                                 n = MAX_DATA_COUNT - tx_strm.avail_out;
 252                                 if (flush != Z_NO_FLUSH) {
 253                                         /*
 254                                          * We have to trim off the last 4
 255                                          * bytes of output when flushing
 256                                          * (they are just 0, 0, ff, ff).
 257                                          */
 258                                         n -= 4;
 259                                 }
 260                                 if (n > 0) {
 261                                         obuf[0] = DEFLATED_DATA + (n >> 8);
 262                                         obuf[1] = n;
 263                                         write_buf(f, obuf, n+2);
 264                                 }
 265                         }
 266                 } while (nb != 0 || tx_strm.avail_out == 0);
 267                 flush_pending = token == -2;
 268         }
 269
 270         if (token == -1) {
 271                 /* end of file - clean up */
 272                 write_byte(f, END_FLAG);
 273         } else if (token != -2) {
 274                 /* Add the data in the current block to the compressor's
 275                  * history and hash table. */
 276                 do {
 277                         /* Break up long sections in the same way that
 278                          * see_deflate_token() does. */
 279                         int32 n1 = toklen > 0xffff ? 0xffff : toklen;
 280                         toklen -= n1;
 281                         tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n1);
 282                         tx_strm.avail_in = n1;
 283                         tx_strm.next_out = (Bytef *) obuf;
 284                         tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 285                         r = deflate(&tx_strm, Z_INSERT_ONLY);
 286                         if (r != Z_OK || tx_strm.avail_in != 0) {
 287                                 rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
 288                                         r, tx_strm.avail_in);
 289                                 exit_cleanup(RERR_STREAMIO);
 290                         }
 291                 } while (toklen > 0);
 292         }
 293 }
 294
 295 /* tells us what the receiver is in the middle of doing */
 296 static enum { r_init, r_idle, r_running, r_inflating, r_inflated } recv_state;
 297
 298 /* for inflating stuff */
 299 static z_stream rx_strm;
 300 static char *cbuf;
 301 static char *dbuf;
 302
 303 /* for decoding runs of tokens */
 304 static int32 rx_token;
 305 static int32 rx_run;
 306
 307 /* Receive a deflated token and inflate it */
 308 static int32 recv_deflated_token(int f, char **data)
 309 {
 310         static int init_done;
 311         static int32 saved_flag;
 312         int32 n, flag;
 313         int r;
 314
 315         for (;;) {
 316                 switch (recv_state) {
 317                 case r_init:
 318                         if (!init_done) {
 319                                 rx_strm.next_out = NULL;
 320                                 rx_strm.zalloc = NULL;
 321                                 rx_strm.zfree = NULL;
 322                                 if (inflateInit2(&rx_strm, -15) != Z_OK) {
 323                                         rprintf(FERROR, "inflate init failed\n");
 324                                         exit_cleanup(RERR_STREAMIO);
 325                                 }
 326                                 if (!(cbuf = new_array(char, MAX_DATA_COUNT))
 327                                     || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE))))
 328                                         out_of_memory("recv_deflated_token");
 329                                 init_done = 1;
 330                         } else {
 331                                 inflateReset(&rx_strm);
 332                         }
 333                         recv_state = r_idle;
 334                         rx_token = 0;
 335                         break;
 336
 337                 case r_idle:
 338                 case r_inflated:
 339                         if (saved_flag) {
 340                                 flag = saved_flag & 0xff;
 341                                 saved_flag = 0;
 342                         } else
 343                                 flag = read_byte(f);
 344                         if ((flag & 0xC0) == DEFLATED_DATA) {
 345                                 n = ((flag & 0x3f) << 8) + read_byte(f);
 346                                 read_buf(f, cbuf, n);
 347                                 rx_strm.next_in = (Bytef *)cbuf;
 348                                 rx_strm.avail_in = n;
 349                                 recv_state = r_inflating;
 350                                 break;
 351                         }
 352                         if (recv_state == r_inflated) {
 353                                 /* check previous inflated stuff ended correctly */
 354                                 rx_strm.avail_in = 0;
 355                                 rx_strm.next_out = (Bytef *)dbuf;
 356                                 rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 357                                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
 358                                 n = AVAIL_OUT_SIZE(CHUNK_SIZE) - rx_strm.avail_out;
 359                                 /*
 360                                  * Z_BUF_ERROR just means no progress was
 361                                  * made, i.e. the decompressor didn't have
 362                                  * any pending output for us.
 363                                  */
 364                                 if (r != Z_OK && r != Z_BUF_ERROR) {
 365                                         rprintf(FERROR, "inflate flush returned %d (%d bytes)\n",
 366                                                 r, n);
 367                                         exit_cleanup(RERR_STREAMIO);
 368                                 }
 369                                 if (n != 0 && r != Z_BUF_ERROR) {
 370                                         /* have to return some more data and
 371                                            save the flag for later. */
 372                                         saved_flag = flag + 0x10000;
 373                                         *data = dbuf;
 374                                         return n;
 375                                 }
 376                                 /*
 377                                  * At this point the decompressor should
 378                                  * be expecting to see the 0, 0, ff, ff bytes.
 379                                  */
 380                                 if (!inflateSyncPoint(&rx_strm)) {
 381                                         rprintf(FERROR, "decompressor lost sync!\n");
 382                                         exit_cleanup(RERR_STREAMIO);
 383                                 }
 384                                 rx_strm.avail_in = 4;
 385                                 rx_strm.next_in = (Bytef *)cbuf;
 386                                 cbuf[0] = cbuf[1] = 0;
 387                                 cbuf[2] = cbuf[3] = 0xff;
 388                                 inflate(&rx_strm, Z_SYNC_FLUSH);
 389                                 recv_state = r_idle;
 390                         }
 391                         if (flag == END_FLAG) {
 392                                 /* that's all folks */
 393                                 recv_state = r_init;
 394                                 return 0;
 395                         }
 396
 397                         /* here we have a token of some kind */
 398                         if (flag & TOKEN_REL) {
 399                                 rx_token += flag & 0x3f;
 400                                 flag >>= 6;
 401                         } else
 402                                 rx_token = read_int(f);
 403                         if (flag & 1) {
 404                                 rx_run = read_byte(f);
 405                                 rx_run += read_byte(f) << 8;
 406                                 recv_state = r_running;
 407                         }
 408                         return -1 - rx_token;
 409
 410                 case r_inflating:
 411                         rx_strm.next_out = (Bytef *)dbuf;
 412                         rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 413                         r = inflate(&rx_strm, Z_NO_FLUSH);
 414                         n = AVAIL_OUT_SIZE(CHUNK_SIZE) - rx_strm.avail_out;
 415                         if (r != Z_OK) {
 416                                 rprintf(FERROR, "inflate returned %d (%d bytes)\n", r, n);
 417                                 exit_cleanup(RERR_STREAMIO);
 418                         }
 419                         if (rx_strm.avail_in == 0)
 420                                 recv_state = r_inflated;
 421                         if (n != 0) {
 422                                 *data = dbuf;
 423                                 return n;
 424                         }
 425                         break;
 426
 427                 case r_running:
 428                         ++rx_token;
 429                         if (--rx_run == 0)
 430                                 recv_state = r_idle;
 431                         return -1 - rx_token;
 432                 }
 433         }
 434 }
 435
 436 /*
 437  * put the data corresponding to a token that we've just returned
 438  * from recv_deflated_token into the decompressor's history buffer.
 439  */
 440 static void see_deflate_token(char *buf, int32 len)
 441 {
 442         int r;
 443         int32 blklen;
 444         unsigned char hdr[5];
 445
 446         rx_strm.avail_in = 0;
 447         blklen = 0;
 448         hdr[0] = 0;
 449         do {
 450                 if (rx_strm.avail_in == 0 && len != 0) {
 451                         if (blklen == 0) {
 452                                 /* Give it a fake stored-block header. */
 453                                 rx_strm.next_in = (Bytef *)hdr;
 454                                 rx_strm.avail_in = 5;
 455                                 blklen = len;
 456                                 if (blklen > 0xffff)
 457                                         blklen = 0xffff;
 458                                 hdr[1] = blklen;
 459                                 hdr[2] = blklen >> 8;
 460                                 hdr[3] = ~hdr[1];
 461                                 hdr[4] = ~hdr[2];
 462                         } else {
 463                                 rx_strm.next_in = (Bytef *)buf;
 464                                 rx_strm.avail_in = blklen;
 465                                 len -= blklen;
 466                                 blklen = 0;
 467                         }
 468                 }
 469                 rx_strm.next_out = (Bytef *)dbuf;
 470                 rx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
 471                 r = inflate(&rx_strm, Z_SYNC_FLUSH);
 472                 if (r != Z_OK) {
 473                         rprintf(FERROR, "inflate (token) returned %d\n", r);
 474                         exit_cleanup(RERR_STREAMIO);
 475                 }
 476         } while (len || rx_strm.avail_out == 0);
 477 }
 478
 479 /**
 480  * Transmit a verbatim buffer of length @p n followed by a token.
 481  * If token == -1 then we have reached EOF
 482  * If n == 0 then don't send a buffer
 483  */
 484 void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
 485                 int32 n, int32 toklen)
 486 {
 487         if (!do_compression)
 488                 simple_send_token(f, token, buf, offset, n);
 489         else
 490                 send_deflated_token(f, token, buf, offset, n, toklen);
 491 }
 492
 493 /*
 494  * receive a token or buffer from the other end. If the reurn value is >0 then
 495  * it is a data buffer of that length, and *data will point at the data.
 496  * if the return value is -i then it represents token i-1
 497  * if the return value is 0 then the end has been reached
 498  */
 499 int32 recv_token(int f, char **data)
 500 {
 501         int tok;
 502
 503         if (!do_compression) {
 504                 tok = simple_recv_token(f,data);
 505         } else {
 506                 tok = recv_deflated_token(f, data);
 507         }
 508         return tok;
 509 }
 510
 511 /*
 512  * look at the data corresponding to a token, if necessary
 513  */
 514 void see_token(char *data, int32 toklen)
 515 {
 516         if (do_compression)
 517                 see_deflate_token(data, toklen);
 518 }