generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int keep_dirlinks;
  30 extern int preserve_links;
  31 extern int am_root;
  32 extern int preserve_devices;
  33 extern int preserve_hard_links;
  34 extern int preserve_perms;
  35 extern int preserve_uid;
  36 extern int preserve_gid;
  37 extern int update_only;
  38 extern int opt_ignore_existing;
  39 extern int csum_length;
  40 extern int ignore_times;
  41 extern int size_only;
  42 extern int io_timeout;
  43 extern int protocol_version;
  44 extern int always_checksum;
  45 extern char *compare_dest;
  46 extern int link_dest;
  47 extern int whole_file;
  48 extern int local_server;
  49 extern int list_only;
  50 extern int read_batch;
  51 extern int only_existing;
  52 extern int orig_umask;
  53 extern int safe_symlinks;
  54 extern unsigned int block_size;
  55
  56 extern struct exclude_list_struct server_exclude_list;
  57
  58
  59 /* choose whether to skip a particular file */
  60 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
  61 {
  62         if (st->st_size != file->length)
  63                 return 0;
  64         if (link_dest) {
  65                 if (preserve_perms
  66                     && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
  67                         return 0;
  68
  69                 if (am_root && preserve_uid && st->st_uid != file->uid)
  70                         return 0;
  71
  72                 if (preserve_gid && file->gid != GID_NONE
  73                     && st->st_gid != file->gid)
  74                         return 0;
  75         }
  76
  77         /* if always checksum is set then we use the checksum instead
  78            of the file time to determine whether to sync */
  79         if (always_checksum && S_ISREG(st->st_mode)) {
  80                 char sum[MD4_SUM_LENGTH];
  81                 char fnamecmpdest[MAXPATHLEN];
  82
  83                 if (compare_dest != NULL) {
  84                         if (access(fname, 0) != 0) {
  85                                 pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  86                                          compare_dest, fname);
  87                                 fname = fnamecmpdest;
  88                         }
  89                 }
  90                 file_checksum(fname,sum,st->st_size);
  91                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  92                                                         : MD4_SUM_LENGTH) == 0;
  93         }
  94
  95         if (size_only)
  96                 return 1;
  97
  98         if (ignore_times)
  99                 return 0;
 100
 101         return cmp_modtime(st->st_mtime, file->modtime) == 0;
 102 }
 103
 104
 105 /*
 106  * NULL sum_struct means we have no checksums
 107  */
 108 void write_sum_head(int f, struct sum_struct *sum)
 109 {
 110         static struct sum_struct null_sum;
 111
 112         if (sum == NULL)
 113                 sum = &null_sum;
 114
 115         write_int(f, sum->count);
 116         write_int(f, sum->blength);
 117         if (protocol_version >= 27)
 118                 write_int(f, sum->s2length);
 119         write_int(f, sum->remainder);
 120 }
 121
 122 /*
 123  * set (initialize) the size entries in the per-file sum_struct
 124  * calculating dynamic block and checksum sizes.
 125  *
 126  * This is only called from generate_and_send_sums() but is a separate
 127  * function to encapsulate the logic.
 128  *
 129  * The block size is a rounded square root of file length.
 130  *
 131  * The checksum size is determined according to:
 132  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 133  * provided by Donovan Baarda which gives a probability of rsync
 134  * algorithm corrupting data and falling back using the whole md4
 135  * checksums.
 136  *
 137  * This might be made one of several selectable heuristics.
 138  */
 139
 140 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 141 {
 142         unsigned int blength;
 143         int s2length;
 144         uint32 c;
 145         uint64 l;
 146
 147         if (block_size) {
 148                 blength = block_size;
 149         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 150                 blength = BLOCK_SIZE;
 151         } else {
 152                 l = len;
 153                 c = 1;
 154                 while (l >>= 2) {
 155                         c <<= 1;
 156                 }
 157                 blength = 0;
 158                 do {
 159                         blength |= c;
 160                         if (len < (uint64)blength * blength)
 161                                 blength &= ~c;
 162                         c >>= 1;
 163                 } while (c >= 8);       /* round to multiple of 8 */
 164                 blength = MAX(blength, BLOCK_SIZE);
 165                 blength = MIN(blength, MAX_MAP_SIZE);
 166         }
 167
 168         if (protocol_version < 27) {
 169                 s2length = csum_length;
 170         } else if (csum_length == SUM_LENGTH) {
 171                 s2length = SUM_LENGTH;
 172         } else {
 173                 int b = BLOCKSUM_BIAS;
 174                 l = len;
 175                 while (l >>= 1) {
 176                         b += 2;
 177                 }
 178                 c = blength;
 179                 while (c >>= 1 && b) {
 180                         b--;
 181                 }
 182                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 183                                                   * subtract rollsum,
 184                                                   * round up
 185                                                   *    --optimize in compiler--
 186                                                   */
 187                 s2length = MAX(s2length, csum_length);
 188                 s2length = MIN(s2length, SUM_LENGTH);
 189         }
 190
 191         sum->flength    = len;
 192         sum->blength    = blength;
 193         sum->s2length   = s2length;
 194         sum->count      = (len + (blength - 1)) / blength;
 195         sum->remainder  = (len % blength);
 196
 197         if (sum->count && verbose > 2) {
 198                 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
 199                         (double)sum->count, sum->remainder, sum->blength,
 200                         sum->s2length, (double)sum->flength);
 201         }
 202 }
 203
 204
 205 /*
 206  * Generate and send a stream of signatures/checksums that describe a buffer
 207  *
 208  * Generate approximately one checksum every block_len bytes.
 209  */
 210 static void generate_and_send_sums(int fd, OFF_T len, int f_out)
 211 {
 212         size_t i;
 213         struct map_struct *mapbuf;
 214         struct sum_struct sum;
 215         OFF_T offset = 0;
 216
 217         sum_sizes_sqroot(&sum, len);
 218
 219         if (len > 0)
 220                 mapbuf = map_file(fd, len, sum.blength);
 221         else
 222                 mapbuf = NULL;
 223
 224         write_sum_head(f_out, &sum);
 225
 226         for (i = 0; i < sum.count; i++) {
 227                 unsigned int n1 = MIN(len, sum.blength);
 228                 char *map = map_ptr(mapbuf, offset, n1);
 229                 uint32 sum1 = get_checksum1(map, n1);
 230                 char sum2[SUM_LENGTH];
 231
 232                 get_checksum2(map, n1, sum2);
 233
 234                 if (verbose > 3) {
 235                         rprintf(FINFO,
 236                                 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
 237                                 (double)i, (double)offset, n1,
 238                                 (unsigned long)sum1);
 239                 }
 240                 write_int(f_out, sum1);
 241                 write_buf(f_out, sum2, sum.s2length);
 242                 len -= n1;
 243                 offset += n1;
 244         }
 245
 246         if (mapbuf)
 247                 unmap_file(mapbuf);
 248 }
 249
 250
 251
 252 /*
 253  * Acts on file number @p i from @p flist, whose name is @p fname.
 254  *
 255  * First fixes up permissions, then generates checksums for the file.
 256  *
 257  * @note This comment was added later by mbp who was trying to work it
 258  * out.  It might be wrong.
 259  */
 260 static void recv_generator(char *fname, struct file_struct *file, int i,
 261                            int f_out)
 262 {
 263         int fd;
 264         STRUCT_STAT st;
 265         int statret;
 266         char *fnamecmp;
 267         char fnamecmpbuf[MAXPATHLEN];
 268
 269         if (list_only)
 270                 return;
 271
 272         if (verbose > 2)
 273                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 274
 275         if (server_exclude_list.head
 276             && check_exclude(&server_exclude_list, fname,
 277                              S_ISDIR(file->mode)) < 0) {
 278                 if (verbose) {
 279                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
 280                                 fname);
 281                 }
 282                 return;
 283         }
 284
 285         statret = link_stat(fname, &st, keep_dirlinks && S_ISDIR(file->mode));
 286
 287         if (only_existing && statret == -1 && errno == ENOENT) {
 288                 /* we only want to update existing files */
 289                 if (verbose > 1)
 290                         rprintf(FINFO, "not creating new file \"%s\"\n", fname);
 291                 return;
 292         }
 293
 294         if (statret == 0 &&
 295             !preserve_perms &&
 296             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 297                 /* if the file exists already and we aren't perserving
 298                  * permissions then act as though the remote end sent
 299                  * us the file permissions we already have */
 300                 file->mode = (file->mode & ~CHMOD_BITS)
 301                            | (st.st_mode & CHMOD_BITS);
 302         }
 303
 304         if (S_ISDIR(file->mode)) {
 305                 /* The file to be received is a directory, so we need
 306                  * to prepare appropriately.  If there is already a
 307                  * file of that name and it is *not* a directory, then
 308                  * we need to delete it.  If it doesn't exist, then
 309                  * recursively create it. */
 310
 311                 if (dry_run)
 312                         return; /* TODO: causes inaccuracies -- fix */
 313                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 314                         if (robust_unlink(fname) != 0) {
 315                                 rsyserr(FERROR, errno,
 316                                         "recv_generator: unlink %s to make room for directory",
 317                                         full_fname(fname));
 318                                 return;
 319                         }
 320                         statret = -1;
 321                 }
 322                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 323                         if (!(relative_paths && errno == ENOENT
 324                             && create_directory_path(fname, orig_umask) == 0
 325                             && do_mkdir(fname, file->mode) == 0)) {
 326                                 rsyserr(FERROR, errno,
 327                                         "recv_generator: mkdir %s failed",
 328                                         full_fname(fname));
 329                         }
 330                 }
 331                 /* f_out is set to -1 when doing final directory-permission
 332                  * and modification-time repair. */
 333                 if (set_perms(fname, file, statret ? NULL : &st, 0)
 334                     && verbose && f_out != -1)
 335                         rprintf(FINFO,"%s/\n",fname);
 336                 return;
 337         }
 338
 339         if (preserve_links && S_ISLNK(file->mode)) {
 340 #if SUPPORT_LINKS
 341                 char lnk[MAXPATHLEN];
 342                 int l;
 343
 344                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
 345                         if (verbose) {
 346                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 347                                         full_fname(fname), file->u.link);
 348                         }
 349                         return;
 350                 }
 351                 if (statret == 0) {
 352                         l = readlink(fname,lnk,MAXPATHLEN-1);
 353                         if (l > 0) {
 354                                 lnk[l] = 0;
 355                                 /* A link already pointing to the
 356                                  * right place -- no further action
 357                                  * required. */
 358                                 if (strcmp(lnk,file->u.link) == 0) {
 359                                         set_perms(fname, file, &st,
 360                                                   PERMS_REPORT);
 361                                         return;
 362                                 }
 363                         }
 364                         /* Not a symlink, so delete whatever's
 365                          * already there and put a new symlink
 366                          * in place. */
 367                         delete_file(fname);
 368                 }
 369                 if (do_symlink(file->u.link,fname) != 0) {
 370                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
 371                                 full_fname(fname), file->u.link);
 372                 } else {
 373                         set_perms(fname,file,NULL,0);
 374                         if (verbose) {
 375                                 rprintf(FINFO,"%s -> %s\n", fname,file->u.link);
 376                         }
 377                 }
 378 #endif
 379                 return;
 380         }
 381
 382 #ifdef HAVE_MKNOD
 383         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 384                 if (statret != 0 ||
 385                     st.st_mode != file->mode ||
 386                     st.st_rdev != file->u.rdev) {
 387                         delete_file(fname);
 388                         if (verbose > 2) {
 389                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 390                                         fname,(int)file->mode,(int)file->u.rdev);
 391                         }
 392                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
 393                                 rsyserr(FERROR, errno, "mknod %s failed",
 394                                         full_fname(fname));
 395                         } else {
 396                                 set_perms(fname,file,NULL,0);
 397                                 if (verbose)
 398                                         rprintf(FINFO,"%s\n",fname);
 399                         }
 400                 } else {
 401                         set_perms(fname, file, &st, PERMS_REPORT);
 402                 }
 403                 return;
 404         }
 405 #endif
 406
 407         if (read_batch)
 408                 return;
 409
 410         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
 411                 return;
 412
 413         if (!S_ISREG(file->mode)) {
 414                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 415                 return;
 416         }
 417
 418         fnamecmp = fname;
 419
 420         if (statret == -1 && compare_dest != NULL) {
 421                 /* try the file at compare_dest instead */
 422                 int saveerrno = errno;
 423                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
 424                 statret = link_stat(fnamecmpbuf, &st, 0);
 425                 if (!S_ISREG(st.st_mode))
 426                         statret = -1;
 427                 if (statret == -1)
 428                         errno = saveerrno;
 429 #if HAVE_LINK
 430                 else if (link_dest && !dry_run) {
 431                         if (do_link(fnamecmpbuf, fname) != 0) {
 432                                 if (verbose > 0) {
 433                                         rsyserr(FINFO, errno, "link %s => %s",
 434                                                 fnamecmpbuf, fname);
 435                                 }
 436                         }
 437                         fnamecmp = fnamecmpbuf;
 438                 }
 439 #endif
 440                 else
 441                         fnamecmp = fnamecmpbuf;
 442         }
 443
 444         if (statret == -1) {
 445                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 446                         return;
 447                 if (errno == ENOENT) {
 448                         write_int(f_out,i);
 449                         if (!dry_run)
 450                                 write_sum_head(f_out, NULL);
 451                 } else if (verbose > 1) {
 452                         rsyserr(FERROR, errno,
 453                                 "recv_generator: failed to open %s",
 454                                 full_fname(fname));
 455                 }
 456                 return;
 457         }
 458
 459         if (!S_ISREG(st.st_mode)) {
 460                 if (delete_file(fname) != 0) {
 461                         return;
 462                 }
 463
 464                 /* now pretend the file didn't exist */
 465                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 466                         return;
 467                 write_int(f_out,i);
 468                 if (!dry_run)
 469                         write_sum_head(f_out, NULL);
 470                 return;
 471         }
 472
 473         if (opt_ignore_existing && fnamecmp == fname) {
 474                 if (verbose > 1)
 475                         rprintf(FINFO,"%s exists\n",fname);
 476                 return;
 477         }
 478
 479         if (update_only && fnamecmp == fname
 480             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 481                 if (verbose > 1)
 482                         rprintf(FINFO,"%s is newer\n",fname);
 483                 return;
 484         }
 485
 486         if (skip_file(fname, file, &st)) {
 487                 if (fnamecmp == fname)
 488                         set_perms(fname, file, &st, PERMS_REPORT);
 489                 return;
 490         }
 491
 492         if (dry_run) {
 493                 write_int(f_out,i);
 494                 return;
 495         }
 496
 497         if (whole_file > 0) {
 498                 write_int(f_out,i);
 499                 write_sum_head(f_out, NULL);
 500                 return;
 501         }
 502
 503         /* open the file */
 504         fd = do_open(fnamecmp, O_RDONLY, 0);
 505
 506         if (fd == -1) {
 507                 rsyserr(FERROR, errno, "failed to open %s, continuing",
 508                         full_fname(fnamecmp));
 509                 /* pretend the file didn't exist */
 510                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 511                         return;
 512                 write_int(f_out,i);
 513                 write_sum_head(f_out, NULL);
 514                 return;
 515         }
 516
 517         if (verbose > 3) {
 518                 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
 519                         (double)st.st_size);
 520         }
 521
 522         if (verbose > 2)
 523                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 524
 525         write_int(f_out,i);
 526         generate_and_send_sums(fd, st.st_size, f_out);
 527
 528         close(fd);
 529 }
 530
 531
 532 void generate_files(int f_out, struct file_list *flist, char *local_name)
 533 {
 534         int i;
 535         int phase = 0;
 536         char fbuf[MAXPATHLEN];
 537
 538         if (verbose > 2) {
 539                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
 540                         (long)getpid(), flist->count);
 541         }
 542
 543         if (verbose >= 2) {
 544                 rprintf(FINFO,
 545                         whole_file > 0
 546                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 547                         : "delta transmission enabled\n");
 548         }
 549
 550         /* we expect to just sit around now, so don't exit on a
 551            timeout. If we really get a timeout then the other process should
 552            exit */
 553         io_timeout = 0;
 554
 555         for (i = 0; i < flist->count; i++) {
 556                 struct file_struct *file = flist->files[i];
 557                 struct file_struct copy;
 558
 559                 if (!file->basename)
 560                         continue;
 561                 /* we need to ensure that any directories we create have writeable
 562                    permissions initially so that we can create the files within
 563                    them. This is then fixed after the files are transferred */
 564                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
 565                         copy = *file;
 566                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 567                          * handling of permissions is strange? */
 568                         copy.mode |= S_IWUSR; /* user write */
 569                         file = &copy;
 570                 }
 571
 572                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 573                                file, i, f_out);
 574         }
 575
 576         phase++;
 577         csum_length = SUM_LENGTH;
 578         ignore_times = 1;
 579
 580         if (verbose > 2)
 581                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 582
 583         write_int(f_out, -1);
 584
 585         /* files can cycle through the system more than once
 586          * to catch initial checksum errors */
 587         while ((i = get_redo_num()) != -1) {
 588                 struct file_struct *file = flist->files[i];
 589                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 590                                file, i, f_out);
 591         }
 592
 593         phase++;
 594         if (verbose > 2)
 595                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 596
 597         write_int(f_out, -1);
 598
 599         if (preserve_hard_links)
 600                 do_hard_links();
 601
 602         /* now we need to fix any directory permissions that were
 603          * modified during the transfer */
 604         for (i = 0; i < flist->count; i++) {
 605                 struct file_struct *file = flist->files[i];
 606                 if (!file->basename || !S_ISDIR(file->mode))
 607                         continue;
 608                 recv_generator(local_name ? local_name : f_name(file),
 609                                file, i, -1);
 610         }
 611
 612         if (verbose > 2)
 613                 rprintf(FINFO,"generate_files finished\n");
 614 }