generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int csum_length;
  36 extern int ignore_times;
  37 extern int size_only;
  38 extern int io_timeout;
  39 extern int protocol_version;
  40 extern int always_checksum;
  41 extern int modify_window;
  42 extern char *compare_dest;
  43 extern int link_dest;
  44
  45
  46 /* choose whether to skip a particular file */
  47 static int skip_file(char *fname,
  48                      struct file_struct *file, STRUCT_STAT *st)
  49 {
  50         if (st->st_size != file->length) {
  51                 return 0;
  52         }
  53         if (link_dest) {
  54                 extern int preserve_perms;
  55                 extern int preserve_uid;
  56                 extern int preserve_gid;
  57
  58                 if(preserve_perms
  59                     && (st->st_mode & ~_S_IFMT) !=  (file->mode & ~_S_IFMT))
  60                         return 0;
  61
  62                 if (preserve_uid && st->st_uid != file->uid)
  63                         return 0;
  64
  65                 if (preserve_gid && st->st_gid != file->gid)
  66                         return 0;
  67         }
  68
  69         /* if always checksum is set then we use the checksum instead
  70            of the file time to determine whether to sync */
  71         if (always_checksum && S_ISREG(st->st_mode)) {
  72                 char sum[MD4_SUM_LENGTH];
  73                 char fnamecmpdest[MAXPATHLEN];
  74
  75                 if (compare_dest != NULL) {
  76                         if (access(fname, 0) != 0) {
  77                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  78                                          compare_dest,fname);
  79                                 fname = fnamecmpdest;
  80                         }
  81                 }
  82                 file_checksum(fname,sum,st->st_size);
  83                 if (protocol_version < 21) {
  84                         return (memcmp(sum,file->sum,2) == 0);
  85                 } else {
  86                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  87                 }
  88         }
  89
  90         if (size_only) {
  91                 return 1;
  92         }
  93
  94         if (ignore_times) {
  95                 return 0;
  96         }
  97
  98         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  99 }
 100
 101
 102 /*
 103  *      NULL sum_struct means we have no checksums
 104  */
 105
 106 void write_sum_head(int f, struct sum_struct *sum)
 107 {
 108         static struct sum_struct null_sum;
 109
 110         if (sum == (struct sum_struct *)NULL)
 111                 sum = &null_sum;
 112
 113         write_int(f, sum->count);
 114         write_int(f, sum->blength);
 115         if (protocol_version >= 27)
 116                 write_int(f, sum->s2length);
 117         write_int(f, sum->remainder);
 118 }
 119
 120 /*
 121  * set (initialize) the size entries in the per-file sum_struct
 122  * calulating dynamic block ans checksum sizes.
 123  *
 124  * This is only called from generate_and_send_sums() but is a seperate
 125  * function to encapsulate the logic.
 126  *
 127  * The block size is a rounded square root of file length.
 128  *
 129  * The checksum size is determined according to:
 130  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 131  * provided by Donovan Baarda which gives a probability of rsync
 132  * algorithm corrupting data and falling back using the whole md4
 133  * checksums.
 134  *
 135  * This might be made one of several selectable heuristics.
 136  */
 137
 138 static void sum_sizes_sqroot_baarda(struct sum_struct *sum, uint64 len)
 139 {
 140         extern int block_size;
 141         int blength, s2length, b;
 142         uint32 c;
 143         uint64 l;
 144
 145         if (block_size) {
 146                 blength = block_size;
 147         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 148                 blength = BLOCK_SIZE;
 149         } else {
 150                 l = len;
 151                 c = 1;
 152                 while (l >>= 2) {
 153                         c <<= 1;
 154                 }
 155                 blength = 0;
 156                 do {
 157                         blength |= c;
 158                         if (len < (uint64)blength * blength)
 159                                 blength &= ~c;
 160                         c >>= 1;
 161                 } while (c >= 8);       /* round to multiple of 8 */
 162                 blength = MAX(blength, BLOCK_SIZE);
 163         }
 164
 165         if (protocol_version < 27) {
 166                 s2length = csum_length;
 167         } else if (csum_length == SUM_LENGTH) {
 168                 s2length = SUM_LENGTH;
 169         } else {
 170                 b = BLOCKSUM_BIAS;
 171                 l = len;
 172                 while (l >>= 1) {
 173                         b += 2;
 174                 }
 175                 c = blength;
 176                 while (c >>= 1 && b) {
 177                         b--;
 178                 }
 179                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 180                                                   * subtract rollsum,
 181                                                   * round up
 182                                                   *    --optimize in compiler--
 183                                                   */
 184                 s2length = MAX(s2length, csum_length);
 185                 s2length = MIN(s2length, SUM_LENGTH);
 186         }
 187
 188         sum->flength    = len;
 189         sum->blength    = blength;
 190         sum->s2length   = s2length;
 191         sum->count      = (len + (blength - 1)) / blength;
 192         sum->remainder  = (len % blength);
 193
 194         if (sum->count && verbose > 2) {
 195                 rprintf(FINFO, "count=%ld rem=%ld blength=%ld s2length=%ld flength=%.0f\n",
 196                         (long) sum->count, (long) sum->remainder,
 197                         (long) sum->blength, (long) sum->s2length,
 198                         (double) sum->flength);
 199         }
 200 }
 201
 202 /**
 203  * Perhaps we want to just send an empty checksum set for this file,
 204  * which will force the whole thing to be literally transferred.
 205  *
 206  * When do we do this?  If the user's explicitly said they
 207  * want the whole thing, or if { they haven't explicitly
 208  * requested a delta, and it's local but not batch mode.}
 209  *
 210  * Whew. */
 211 static BOOL disable_deltas_p(void)
 212 {
 213         extern int whole_file;
 214         extern int local_server;
 215         extern int write_batch;
 216
 217         if (whole_file > 0)
 218                 return True;
 219         if (whole_file == 0 || write_batch)
 220                 return False;
 221         return local_server;
 222 }
 223
 224
 225 /*
 226  * Generate and send a stream of signatures/checksums that describe a buffer
 227  *
 228  * Generate approximately one checksum every block_len bytes.
 229  */
 230 static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out)
 231 {
 232         size_t i;
 233         struct sum_struct sum;
 234         OFF_T offset = 0;
 235
 236         sum_sizes_sqroot_baarda(&sum, len);
 237
 238         write_sum_head(f_out, &sum);
 239
 240         for (i = 0; i < sum.count; i++) {
 241                 int n1 = MIN(len, sum.blength);
 242                 char *map = map_ptr(buf, offset, n1);
 243                 uint32 sum1 = get_checksum1(map, n1);
 244                 char sum2[SUM_LENGTH];
 245
 246                 get_checksum2(map, n1, sum2);
 247
 248                 if (verbose > 3) {
 249                         rprintf(FINFO,
 250                                 "chunk[%ld] offset=%.0f len=%d sum1=%08lx\n",
 251                                 (long)i,(double)offset,n1,(unsigned long)sum1);
 252                 }
 253                 write_int(f_out, sum1);
 254                 write_buf(f_out, sum2, sum.s2length);
 255                 len -= n1;
 256                 offset += n1;
 257         }
 258 }
 259
 260
 261
 262 /**
 263  * Acts on file number @p i from @p flist, whose name is @p fname.
 264  *
 265  * First fixes up permissions, then generates checksums for the file.
 266  *
 267  * @note This comment was added later by mbp who was trying to work it
 268  * out.  It might be wrong.
 269  **/
 270 void recv_generator(char *fname, struct file_list *flist, int i, int f_out)
 271 {
 272         int fd;
 273         STRUCT_STAT st;
 274         struct map_struct *buf;
 275         int statret;
 276         struct file_struct *file = flist->files[i];
 277         char *fnamecmp;
 278         char fnamecmpbuf[MAXPATHLEN];
 279         extern char *compare_dest;
 280         extern int list_only;
 281         extern int preserve_perms;
 282         extern int only_existing;
 283         extern int orig_umask;
 284
 285         if (list_only) return;
 286
 287         if (verbose > 2)
 288                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 289
 290         statret = link_stat(fname,&st);
 291
 292         if (only_existing && statret == -1 && errno == ENOENT) {
 293                 /* we only want to update existing files */
 294                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 295                 return;
 296         }
 297
 298         if (statret == 0 &&
 299             !preserve_perms &&
 300             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 301                 /* if the file exists already and we aren't perserving
 302                  * permissions then act as though the remote end sent
 303                  * us the file permissions we already have */
 304                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 305         }
 306
 307         if (S_ISDIR(file->mode)) {
 308                 /* The file to be received is a directory, so we need
 309                  * to prepare appropriately.  If there is already a
 310                  * file of that name and it is *not* a directory, then
 311                  * we need to delete it.  If it doesn't exist, then
 312                  * recursively create it. */
 313
 314                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 315                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 316                         if (robust_unlink(fname) != 0) {
 317                                 rprintf(FERROR,
 318                                         "recv_generator: unlink %s to make room for directory: %s\n",
 319                                         full_fname(fname), strerror(errno));
 320                                 return;
 321                         }
 322                         statret = -1;
 323                 }
 324                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 325                         if (!(relative_paths && errno==ENOENT &&
 326                               create_directory_path(fname, orig_umask)==0 &&
 327                               do_mkdir(fname,file->mode)==0)) {
 328                                 rprintf(FERROR, "recv_generator: mkdir %s failed: %s\n",
 329                                         full_fname(fname), strerror(errno));
 330                         }
 331                 }
 332                 /* f_out is set to -1 when doing final directory
 333                    permission and modification time repair */
 334                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 335                         rprintf(FINFO,"%s/\n",fname);
 336                 return;
 337         }
 338
 339         if (preserve_links && S_ISLNK(file->mode)) {
 340 #if SUPPORT_LINKS
 341                 char lnk[MAXPATHLEN];
 342                 int l;
 343                 extern int safe_symlinks;
 344
 345                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 346                         if (verbose) {
 347                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 348                                         full_fname(fname), file->link);
 349                         }
 350                         return;
 351                 }
 352                 if (statret == 0) {
 353                         l = readlink(fname,lnk,MAXPATHLEN-1);
 354                         if (l > 0) {
 355                                 lnk[l] = 0;
 356                                 /* A link already pointing to the
 357                                  * right place -- no further action
 358                                  * required. */
 359                                 if (strcmp(lnk,file->link) == 0) {
 360                                         set_perms(fname,file,&st,1);
 361                                         return;
 362                                 }
 363                         }
 364                         /* Not a symlink, so delete whatever's
 365                          * already there and put a new symlink
 366                          * in place. */
 367                         delete_file(fname);
 368                 }
 369                 if (do_symlink(file->link,fname) != 0) {
 370                         rprintf(FERROR, "symlink %s -> \"%s\" failed: %s\n",
 371                                 full_fname(fname), file->link, strerror(errno));
 372                 } else {
 373                         set_perms(fname,file,NULL,0);
 374                         if (verbose) {
 375                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 376                         }
 377                 }
 378 #endif
 379                 return;
 380         }
 381
 382 #ifdef HAVE_MKNOD
 383         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 384                 if (statret != 0 ||
 385                     st.st_mode != file->mode ||
 386                     (DEV64_T)st.st_rdev != file->rdev) {
 387                         delete_file(fname);
 388                         if (verbose > 2)
 389                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 390                                         fname,(int)file->mode,(int)file->rdev);
 391                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 392                                 rprintf(FERROR, "mknod %s failed: %s\n",
 393                                         full_fname(fname), strerror(errno));
 394                         } else {
 395                                 set_perms(fname,file,NULL,0);
 396                                 if (verbose)
 397                                         rprintf(FINFO,"%s\n",fname);
 398                         }
 399                 } else {
 400                         set_perms(fname,file,&st,1);
 401                 }
 402                 return;
 403         }
 404 #endif
 405
 406         if (preserve_hard_links && check_hard_link(file)) {
 407                 if (verbose > 1)
 408                         rprintf(FINFO, "recv_generator: \"%s\" is a hard link\n",f_name(file));
 409                 return;
 410         }
 411
 412         if (!S_ISREG(file->mode)) {
 413                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 414                 return;
 415         }
 416
 417         fnamecmp = fname;
 418
 419         if ((statret == -1) && (compare_dest != NULL)) {
 420                 /* try the file at compare_dest instead */
 421                 int saveerrno = errno;
 422                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 423                 statret = link_stat(fnamecmpbuf,&st);
 424                 if (!S_ISREG(st.st_mode))
 425                         statret = -1;
 426                 if (statret == -1)
 427                         errno = saveerrno;
 428 #if HAVE_LINK
 429                 else if (link_dest && !dry_run) {
 430                         if (do_link(fnamecmpbuf, fname) != 0) {
 431                                 if (verbose > 0)
 432                                         rprintf(FINFO,"link %s => %s : %s\n",
 433                                                 fnamecmpbuf,
 434                                                 fname,
 435                                                 strerror(errno));
 436                         }
 437                         fnamecmp = fnamecmpbuf;
 438                 }
 439 #endif
 440                 else
 441                         fnamecmp = fnamecmpbuf;
 442         }
 443
 444         if (statret == -1) {
 445                 if (errno == ENOENT) {
 446                         write_int(f_out,i);
 447                         if (!dry_run) write_sum_head(f_out, NULL);
 448                 } else if (verbose > 1) {
 449                         rprintf(FERROR,
 450                                 "recv_generator: failed to open %s: %s\n",
 451                                 full_fname(fname), strerror(errno));
 452                 }
 453                 return;
 454         }
 455
 456         if (!S_ISREG(st.st_mode)) {
 457                 if (delete_file(fname) != 0) {
 458                         return;
 459                 }
 460
 461                 /* now pretend the file didn't exist */
 462                 write_int(f_out,i);
 463                 if (!dry_run) write_sum_head(f_out, NULL);
 464                 return;
 465         }
 466
 467         if (opt_ignore_existing && fnamecmp == fname) {
 468                 if (verbose > 1)
 469                         rprintf(FINFO,"%s exists\n",fname);
 470                 return;
 471         }
 472
 473         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 474                 if (verbose > 1)
 475                         rprintf(FINFO,"%s is newer\n",fname);
 476                 return;
 477         }
 478
 479         if (skip_file(fname, file, &st)) {
 480                 if (fnamecmp == fname)
 481                         set_perms(fname,file,&st,1);
 482                 return;
 483         }
 484
 485         if (dry_run) {
 486                 write_int(f_out,i);
 487                 return;
 488         }
 489
 490         if (disable_deltas_p()) {
 491                 write_int(f_out,i);
 492                 write_sum_head(f_out, NULL);
 493                 return;
 494         }
 495
 496         /* open the file */
 497         fd = do_open(fnamecmp, O_RDONLY, 0);
 498
 499         if (fd == -1) {
 500                 rprintf(FERROR, "failed to open %s, continuing: %s\n",
 501                         full_fname(fnamecmp), strerror(errno));
 502                 /* pretend the file didn't exist */
 503                 write_int(f_out,i);
 504                 write_sum_head(f_out, NULL);
 505                 return;
 506         }
 507
 508         if (st.st_size > 0) {
 509                 buf = map_file(fd,st.st_size);
 510         } else {
 511                 buf = NULL;
 512         }
 513
 514         if (verbose > 3)
 515                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 516
 517         if (verbose > 2)
 518                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 519
 520         write_int(f_out,i);
 521         generate_and_send_sums(buf, st.st_size, f_out);
 522
 523         close(fd);
 524         if (buf) unmap_file(buf);
 525 }
 526
 527
 528
 529 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 530 {
 531         int i;
 532         int phase=0;
 533
 534         if (verbose > 2)
 535                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 536                         (int)getpid(),flist->count);
 537
 538         if (verbose >= 2) {
 539                 rprintf(FINFO,
 540                         disable_deltas_p()
 541                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 542                         : "delta transmission enabled\n");
 543         }
 544
 545         /* we expect to just sit around now, so don't exit on a
 546            timeout. If we really get a timeout then the other process should
 547            exit */
 548         io_timeout = 0;
 549
 550         for (i = 0; i < flist->count; i++) {
 551                 struct file_struct *file = flist->files[i];
 552                 mode_t saved_mode = file->mode;
 553                 if (!file->basename) continue;
 554
 555                 /* we need to ensure that any directories we create have writeable
 556                    permissions initially so that we can create the files within
 557                    them. This is then fixed after the files are transferred */
 558                 if (!am_root && S_ISDIR(file->mode)) {
 559                         file->mode |= S_IWUSR; /* user write */
 560                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 561                          * handling of permissions is strange? */
 562                 }
 563
 564                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 565
 566                 file->mode = saved_mode;
 567         }
 568
 569         phase++;
 570         csum_length = SUM_LENGTH;
 571         ignore_times=1;
 572
 573         if (verbose > 2)
 574                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 575
 576         write_int(f,-1);
 577
 578         /* files can cycle through the system more than once
 579          * to catch initial checksum errors */
 580         for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 581                 struct file_struct *file = flist->files[i];
 582                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 583         }
 584
 585         phase++;
 586         if (verbose > 2)
 587                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 588
 589         write_int(f,-1);
 590 }