generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int keep_dirlinks;
  30 extern int preserve_links;
  31 extern int am_root;
  32 extern int preserve_devices;
  33 extern int preserve_hard_links;
  34 extern int preserve_perms;
  35 extern int preserve_uid;
  36 extern int preserve_gid;
  37 extern int update_only;
  38 extern int opt_ignore_existing;
  39 extern int inplace;
  40 extern int make_backups;
  41 extern int csum_length;
  42 extern int ignore_times;
  43 extern int size_only;
  44 extern OFF_T max_size;
  45 extern int io_timeout;
  46 extern int protocol_version;
  47 extern int always_checksum;
  48 extern char *partial_dir;
  49 extern char *compare_dest;
  50 extern int link_dest;
  51 extern int whole_file;
  52 extern int local_server;
  53 extern int list_only;
  54 extern int read_batch;
  55 extern int only_existing;
  56 extern int orig_umask;
  57 extern int safe_symlinks;
  58 extern unsigned int block_size;
  59
  60 extern struct exclude_list_struct server_exclude_list;
  61
  62
  63 /* choose whether to skip a particular file */
  64 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
  65 {
  66         if (st->st_size != file->length)
  67                 return 0;
  68         if (link_dest) {
  69                 if (preserve_perms
  70                     && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
  71                         return 0;
  72
  73                 if (am_root && preserve_uid && st->st_uid != file->uid)
  74                         return 0;
  75
  76                 if (preserve_gid && file->gid != GID_NONE
  77                     && st->st_gid != file->gid)
  78                         return 0;
  79         }
  80
  81         /* if always checksum is set then we use the checksum instead
  82            of the file time to determine whether to sync */
  83         if (always_checksum && S_ISREG(st->st_mode)) {
  84                 char sum[MD4_SUM_LENGTH];
  85                 file_checksum(fname,sum,st->st_size);
  86                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  87                                                         : MD4_SUM_LENGTH) == 0;
  88         }
  89
  90         if (size_only)
  91                 return 1;
  92
  93         if (ignore_times)
  94                 return 0;
  95
  96         return cmp_modtime(st->st_mtime, file->modtime) == 0;
  97 }
  98
  99
 100 /*
 101  * NULL sum_struct means we have no checksums
 102  */
 103 void write_sum_head(int f, struct sum_struct *sum)
 104 {
 105         static struct sum_struct null_sum;
 106
 107         if (sum == NULL)
 108                 sum = &null_sum;
 109
 110         write_int(f, sum->count);
 111         write_int(f, sum->blength);
 112         if (protocol_version >= 27)
 113                 write_int(f, sum->s2length);
 114         write_int(f, sum->remainder);
 115 }
 116
 117 /*
 118  * set (initialize) the size entries in the per-file sum_struct
 119  * calculating dynamic block and checksum sizes.
 120  *
 121  * This is only called from generate_and_send_sums() but is a separate
 122  * function to encapsulate the logic.
 123  *
 124  * The block size is a rounded square root of file length.
 125  *
 126  * The checksum size is determined according to:
 127  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 128  * provided by Donovan Baarda which gives a probability of rsync
 129  * algorithm corrupting data and falling back using the whole md4
 130  * checksums.
 131  *
 132  * This might be made one of several selectable heuristics.
 133  */
 134
 135 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 136 {
 137         unsigned int blength;
 138         int s2length;
 139         uint32 c;
 140         uint64 l;
 141
 142         if (block_size) {
 143                 blength = block_size;
 144         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 145                 blength = BLOCK_SIZE;
 146         } else {
 147                 l = len;
 148                 c = 1;
 149                 while (l >>= 2) {
 150                         c <<= 1;
 151                 }
 152                 blength = 0;
 153                 do {
 154                         blength |= c;
 155                         if (len < (uint64)blength * blength)
 156                                 blength &= ~c;
 157                         c >>= 1;
 158                 } while (c >= 8);       /* round to multiple of 8 */
 159                 blength = MAX(blength, BLOCK_SIZE);
 160         }
 161
 162         if (protocol_version < 27) {
 163                 s2length = csum_length;
 164         } else if (csum_length == SUM_LENGTH) {
 165                 s2length = SUM_LENGTH;
 166         } else {
 167                 int b = BLOCKSUM_BIAS;
 168                 l = len;
 169                 while (l >>= 1) {
 170                         b += 2;
 171                 }
 172                 c = blength;
 173                 while (c >>= 1 && b) {
 174                         b--;
 175                 }
 176                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 177                                                   * subtract rollsum,
 178                                                   * round up
 179                                                   *    --optimize in compiler--
 180                                                   */
 181                 s2length = MAX(s2length, csum_length);
 182                 s2length = MIN(s2length, SUM_LENGTH);
 183         }
 184
 185         sum->flength    = len;
 186         sum->blength    = blength;
 187         sum->s2length   = s2length;
 188         sum->count      = (len + (blength - 1)) / blength;
 189         sum->remainder  = (len % blength);
 190
 191         if (sum->count && verbose > 2) {
 192                 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
 193                         (double)sum->count, sum->remainder, sum->blength,
 194                         sum->s2length, (double)sum->flength);
 195         }
 196 }
 197
 198
 199 /*
 200  * Generate and send a stream of signatures/checksums that describe a buffer
 201  *
 202  * Generate approximately one checksum every block_len bytes.
 203  */
 204 static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 205 {
 206         size_t i;
 207         struct map_struct *mapbuf;
 208         struct sum_struct sum;
 209         OFF_T offset = 0;
 210
 211         sum_sizes_sqroot(&sum, len);
 212
 213         if (len > 0)
 214                 mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
 215         else
 216                 mapbuf = NULL;
 217
 218         write_sum_head(f_out, &sum);
 219
 220         for (i = 0; i < sum.count; i++) {
 221                 unsigned int n1 = MIN(len, sum.blength);
 222                 char *map = map_ptr(mapbuf, offset, n1);
 223                 uint32 sum1 = get_checksum1(map, n1);
 224                 char sum2[SUM_LENGTH];
 225
 226                 if (f_copy >= 0)
 227                         full_write(f_copy, map, n1);
 228
 229                 get_checksum2(map, n1, sum2);
 230
 231                 if (verbose > 3) {
 232                         rprintf(FINFO,
 233                                 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
 234                                 (double)i, (double)offset, n1,
 235                                 (unsigned long)sum1);
 236                 }
 237                 write_int(f_out, sum1);
 238                 write_buf(f_out, sum2, sum.s2length);
 239                 len -= n1;
 240                 offset += n1;
 241         }
 242
 243         if (mapbuf)
 244                 unmap_file(mapbuf);
 245 }
 246
 247
 248
 249 /*
 250  * Acts on file number @p i from @p flist, whose name is @p fname.
 251  *
 252  * First fixes up permissions, then generates checksums for the file.
 253  *
 254  * @note This comment was added later by mbp who was trying to work it
 255  * out.  It might be wrong.
 256  */
 257 static void recv_generator(char *fname, struct file_struct *file, int i,
 258                            int f_out, int f_out_name)
 259 {
 260         int fd = -1, f_copy = -1;
 261         STRUCT_STAT st, partial_st;
 262         struct file_struct *back_file = NULL;
 263         int statret, stat_errno;
 264         char *fnamecmp, *partialptr, *backupptr = NULL;
 265         char fnamecmpbuf[MAXPATHLEN];
 266         uchar fnamecmp_type;
 267
 268         if (list_only)
 269                 return;
 270
 271         if (verbose > 2)
 272                 rprintf(FINFO, "recv_generator(%s,%d)\n", safe_fname(fname), i);
 273
 274         if (server_exclude_list.head
 275             && check_exclude(&server_exclude_list, fname,
 276                              S_ISDIR(file->mode)) < 0) {
 277                 if (verbose) {
 278                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
 279                                 safe_fname(fname));
 280                 }
 281                 return;
 282         }
 283
 284         if (dry_run > 1) {
 285                 statret = -1;
 286                 stat_errno = ENOENT;
 287         } else {
 288                 statret = link_stat(fname, &st,
 289                                     keep_dirlinks && S_ISDIR(file->mode));
 290                 stat_errno = errno;
 291         }
 292
 293         if (only_existing && statret == -1 && stat_errno == ENOENT) {
 294                 /* we only want to update existing files */
 295                 if (verbose > 1) {
 296                         rprintf(FINFO, "not creating new file \"%s\"\n",
 297                                 safe_fname(fname));
 298                 }
 299                 return;
 300         }
 301
 302         if (statret == 0 && !preserve_perms
 303             && S_ISDIR(st.st_mode) == S_ISDIR(file->mode)) {
 304                 /* if the file exists already and we aren't perserving
 305                  * permissions then act as though the remote end sent
 306                  * us the file permissions we already have */
 307                 file->mode = (file->mode & ~CHMOD_BITS)
 308                            | (st.st_mode & CHMOD_BITS);
 309         }
 310
 311         if (S_ISDIR(file->mode)) {
 312                 /* The file to be received is a directory, so we need
 313                  * to prepare appropriately.  If there is already a
 314                  * file of that name and it is *not* a directory, then
 315                  * we need to delete it.  If it doesn't exist, then
 316                  * recursively create it. */
 317
 318                 if (dry_run)
 319                         return; /* TODO: causes inaccuracies -- fix */
 320                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 321                         if (robust_unlink(fname) != 0) {
 322                                 rsyserr(FERROR, errno,
 323                                         "recv_generator: unlink %s to make room for directory",
 324                                         full_fname(fname));
 325                                 return;
 326                         }
 327                         statret = -1;
 328                 }
 329                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 330                         if (!(relative_paths && errno == ENOENT
 331                             && create_directory_path(fname, orig_umask) == 0
 332                             && do_mkdir(fname, file->mode) == 0)) {
 333                                 rsyserr(FERROR, errno,
 334                                         "recv_generator: mkdir %s failed",
 335                                         full_fname(fname));
 336                         }
 337                 }
 338                 /* f_out is set to -1 when doing final directory-permission
 339                  * and modification-time repair. */
 340                 if (set_perms(fname, file, statret ? NULL : &st, 0)
 341                     && verbose && f_out != -1)
 342                         rprintf(FINFO, "%s/\n", safe_fname(fname));
 343                 return;
 344         } else if (max_size && file->length > max_size) {
 345                 if (verbose > 1)
 346                         rprintf(FINFO, "%s is over max-size\n", fname);
 347                 return;
 348         }
 349
 350         if (preserve_links && S_ISLNK(file->mode)) {
 351 #if SUPPORT_LINKS
 352                 char lnk[MAXPATHLEN];
 353                 int l;
 354
 355                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
 356                         if (verbose) {
 357                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 358                                         full_fname(fname), file->u.link);
 359                         }
 360                         return;
 361                 }
 362                 if (statret == 0) {
 363                         l = readlink(fname,lnk,MAXPATHLEN-1);
 364                         if (l > 0) {
 365                                 lnk[l] = 0;
 366                                 /* A link already pointing to the
 367                                  * right place -- no further action
 368                                  * required. */
 369                                 if (strcmp(lnk,file->u.link) == 0) {
 370                                         set_perms(fname, file, &st,
 371                                                   PERMS_REPORT);
 372                                         return;
 373                                 }
 374                         }
 375                         /* Not a symlink, so delete whatever's
 376                          * already there and put a new symlink
 377                          * in place. */
 378                         delete_file(fname);
 379                 }
 380                 if (do_symlink(file->u.link,fname) != 0) {
 381                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
 382                                 full_fname(fname), safe_fname(file->u.link));
 383                 } else {
 384                         set_perms(fname,file,NULL,0);
 385                         if (verbose) {
 386                                 rprintf(FINFO, "%s -> %s\n", safe_fname(fname),
 387                                         safe_fname(file->u.link));
 388                         }
 389                 }
 390 #endif
 391                 return;
 392         }
 393
 394         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 395                 if (statret != 0 ||
 396                     st.st_mode != file->mode ||
 397                     st.st_rdev != file->u.rdev) {
 398                         delete_file(fname);
 399                         if (verbose > 2) {
 400                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 401                                         safe_fname(fname),
 402                                         (int)file->mode, (int)file->u.rdev);
 403                         }
 404                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
 405                                 rsyserr(FERROR, errno, "mknod %s failed",
 406                                         full_fname(fname));
 407                         } else {
 408                                 set_perms(fname,file,NULL,0);
 409                                 if (verbose) {
 410                                         rprintf(FINFO, "%s\n",
 411                                                 safe_fname(fname));
 412                                 }
 413                         }
 414                 } else {
 415                         set_perms(fname, file, &st, PERMS_REPORT);
 416                 }
 417                 return;
 418         }
 419
 420         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
 421                 return;
 422
 423         if (!S_ISREG(file->mode)) {
 424                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",
 425                         safe_fname(fname));
 426                 return;
 427         }
 428
 429         fnamecmp = fname;
 430         fnamecmp_type = FNAMECMP_FNAME;
 431
 432         if (statret == -1 && compare_dest != NULL) {
 433                 /* try the file at compare_dest instead */
 434                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
 435                 if (link_stat(fnamecmpbuf, &st, 0) == 0
 436                     && S_ISREG(st.st_mode)) {
 437 #if HAVE_LINK
 438                         if (link_dest && !dry_run) {
 439                                 if (do_link(fnamecmpbuf, fname) < 0) {
 440                                         if (verbose) {
 441                                                 rsyserr(FINFO, errno,
 442                                                         "link %s => %s",
 443                                                         fnamecmpbuf,
 444                                                         safe_fname(fname));
 445                                         }
 446                                         fnamecmp = fnamecmpbuf;
 447                                         fnamecmp_type = FNAMECMP_CMPDEST;
 448                                 }
 449                         } else
 450 #endif
 451                         {
 452                                 fnamecmp = fnamecmpbuf;
 453                                 fnamecmp_type = FNAMECMP_CMPDEST;
 454                         }
 455                         statret = 0;
 456                 }
 457         }
 458
 459         if (statret == 0 && !S_ISREG(st.st_mode)) {
 460                 if (delete_file(fname) != 0)
 461                         return;
 462                 statret = -1;
 463                 stat_errno = ENOENT;
 464         }
 465
 466         if (partial_dir && (partialptr = partial_dir_fname(fname))
 467             && link_stat(partialptr, &partial_st, 0) == 0
 468             && S_ISREG(partial_st.st_mode)) {
 469                 if (statret == -1)
 470                         goto prepare_to_open;
 471         } else
 472                 partialptr = NULL;
 473
 474         if (statret == -1) {
 475                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 476                         return;
 477                 if (stat_errno == ENOENT)
 478                         goto notify_others;
 479                 if (verbose > 1) {
 480                         rsyserr(FERROR, stat_errno,
 481                                 "recv_generator: failed to stat %s",
 482                                 full_fname(fname));
 483                 }
 484                 return;
 485         }
 486
 487         if (opt_ignore_existing && fnamecmp_type == FNAMECMP_FNAME) {
 488                 if (verbose > 1)
 489                         rprintf(FINFO, "%s exists\n", safe_fname(fname));
 490                 return;
 491         }
 492
 493         if (update_only && fnamecmp_type == FNAMECMP_FNAME
 494             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 495                 if (verbose > 1)
 496                         rprintf(FINFO, "%s is newer\n", safe_fname(fname));
 497                 return;
 498         }
 499
 500         if (skip_file(fnamecmp, file, &st)) {
 501                 if (fnamecmp_type == FNAMECMP_FNAME)
 502                         set_perms(fname, file, &st, PERMS_REPORT);
 503                 return;
 504         }
 505
 506 prepare_to_open:
 507         if (dry_run || whole_file > 0) {
 508                 statret = -1;
 509                 goto notify_others;
 510         }
 511         if (read_batch)
 512                 goto notify_others;
 513
 514         if (partialptr) {
 515                 st = partial_st;
 516                 fnamecmp = partialptr;
 517                 fnamecmp_type = FNAMECMP_PARTIAL_DIR;
 518         }
 519
 520         /* open the file */
 521         fd = do_open(fnamecmp, O_RDONLY, 0);
 522
 523         if (fd == -1) {
 524                 rsyserr(FERROR, errno, "failed to open %s, continuing",
 525                         full_fname(fnamecmp));
 526             pretend_missing:
 527                 /* pretend the file didn't exist */
 528                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 529                         return;
 530                 statret = -1;
 531                 goto notify_others;
 532         }
 533
 534         if (inplace && make_backups) {
 535                 if (!(backupptr = get_backup_name(fname))) {
 536                         close(fd);
 537                         return;
 538                 }
 539                 if (!(back_file = make_file(fname, NULL, NO_EXCLUDES))) {
 540                         close(fd);
 541                         goto pretend_missing;
 542                 }
 543                 if (robust_unlink(backupptr) && errno != ENOENT) {
 544                         rsyserr(FERROR, errno, "unlink %s",
 545                                 full_fname(backupptr));
 546                         free(back_file);
 547                         close(fd);
 548                         return;
 549                 }
 550                 if ((f_copy = do_open(backupptr,
 551                     O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
 552                         rsyserr(FERROR, errno, "open %s",
 553                                 full_fname(backupptr));
 554                         free(back_file);
 555                         close(fd);
 556                         return;
 557                 }
 558                 fnamecmp_type = FNAMECMP_BACKUP;
 559         }
 560
 561         if (verbose > 3) {
 562                 rprintf(FINFO, "gen mapped %s of size %.0f\n",
 563                         safe_fname(fnamecmp), (double)st.st_size);
 564         }
 565
 566         if (verbose > 2)
 567                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 568
 569 notify_others:
 570         write_int(f_out, i);
 571         if (f_out_name >= 0)
 572                 write_byte(f_out_name, fnamecmp_type);
 573
 574         if (dry_run || read_batch)
 575                 return;
 576
 577         if (statret == 0) {
 578                 generate_and_send_sums(fd, st.st_size, f_out, f_copy);
 579
 580                 if (f_copy >= 0) {
 581                         close(f_copy);
 582                         set_perms(backupptr, back_file, NULL, 0);
 583                         if (verbose > 1) {
 584                                 rprintf(FINFO, "backed up %s to %s\n",
 585                                         fname, backupptr);
 586                         }
 587                         free(back_file);
 588                 }
 589
 590                 close(fd);
 591         } else
 592                 write_sum_head(f_out, NULL);
 593 }
 594
 595
 596 void generate_files(int f_out, struct file_list *flist, char *local_name,
 597                     int f_out_name)
 598 {
 599         int i;
 600         int phase = 0;
 601         char fbuf[MAXPATHLEN];
 602
 603         if (verbose > 2) {
 604                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
 605                         (long)getpid(), flist->count);
 606         }
 607
 608         if (verbose >= 2) {
 609                 rprintf(FINFO,
 610                         whole_file > 0
 611                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 612                         : "delta transmission enabled\n");
 613         }
 614
 615         /* we expect to just sit around now, so don't exit on a
 616            timeout. If we really get a timeout then the other process should
 617            exit */
 618         io_timeout = 0;
 619
 620         for (i = 0; i < flist->count; i++) {
 621                 struct file_struct *file = flist->files[i];
 622                 struct file_struct copy;
 623
 624                 if (!file->basename)
 625                         continue;
 626                 /* we need to ensure that any directories we create have writeable
 627                    permissions initially so that we can create the files within
 628                    them. This is then fixed after the files are transferred */
 629                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
 630                         copy = *file;
 631                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 632                          * handling of permissions is strange? */
 633                         copy.mode |= S_IWUSR; /* user write */
 634                         file = &copy;
 635                 }
 636
 637                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 638                                file, i, f_out, f_out_name);
 639         }
 640
 641         phase++;
 642         csum_length = SUM_LENGTH;
 643         ignore_times = 1;
 644
 645         if (verbose > 2)
 646                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 647
 648         write_int(f_out, -1);
 649
 650         /* files can cycle through the system more than once
 651          * to catch initial checksum errors */
 652         while ((i = get_redo_num()) != -1) {
 653                 struct file_struct *file = flist->files[i];
 654                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 655                                file, i, f_out, f_out_name);
 656         }
 657
 658         phase++;
 659         if (verbose > 2)
 660                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 661
 662         write_int(f_out, -1);
 663
 664         if (preserve_hard_links)
 665                 do_hard_links();
 666
 667         /* now we need to fix any directory permissions that were
 668          * modified during the transfer */
 669         for (i = 0; i < flist->count; i++) {
 670                 struct file_struct *file = flist->files[i];
 671                 if (!file->basename || !S_ISDIR(file->mode))
 672                         continue;
 673                 recv_generator(local_name ? local_name : f_name(file),
 674                                file, i, -1, -1);
 675         }
 676
 677         if (verbose > 2)
 678                 rprintf(FINFO,"generate_files finished\n");
 679 }