generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int keep_dirlinks;
  30 extern int preserve_links;
  31 extern int am_root;
  32 extern int preserve_devices;
  33 extern int preserve_hard_links;
  34 extern int preserve_perms;
  35 extern int preserve_uid;
  36 extern int preserve_gid;
  37 extern int update_only;
  38 extern int opt_ignore_existing;
  39 extern int csum_length;
  40 extern int ignore_times;
  41 extern int size_only;
  42 extern int io_timeout;
  43 extern int protocol_version;
  44 extern int always_checksum;
  45 extern char *compare_dest;
  46 extern int link_dest;
  47 extern int whole_file;
  48 extern int local_server;
  49 extern int list_only;
  50 extern int read_batch;
  51 extern int only_existing;
  52 extern int orig_umask;
  53 extern int safe_symlinks;
  54 extern unsigned int block_size;
  55
  56 extern struct exclude_list_struct server_exclude_list;
  57
  58
  59 /* choose whether to skip a particular file */
  60 static int skip_file(char *fname, struct file_struct *file, STRUCT_STAT *st)
  61 {
  62         if (st->st_size != file->length)
  63                 return 0;
  64         if (link_dest) {
  65                 if (preserve_perms
  66                     && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
  67                         return 0;
  68
  69                 if (am_root && preserve_uid && st->st_uid != file->uid)
  70                         return 0;
  71
  72                 if (preserve_gid && file->gid != GID_NONE
  73                     && st->st_gid != file->gid)
  74                         return 0;
  75         }
  76
  77         /* if always checksum is set then we use the checksum instead
  78            of the file time to determine whether to sync */
  79         if (always_checksum && S_ISREG(st->st_mode)) {
  80                 char sum[MD4_SUM_LENGTH];
  81                 char fnamecmpdest[MAXPATHLEN];
  82
  83                 if (compare_dest != NULL) {
  84                         if (access(fname, 0) != 0) {
  85                                 pathjoin(fnamecmpdest, sizeof fnamecmpdest,
  86                                          compare_dest, fname);
  87                                 fname = fnamecmpdest;
  88                         }
  89                 }
  90                 file_checksum(fname,sum,st->st_size);
  91                 return memcmp(sum, file->u.sum, protocol_version < 21 ? 2
  92                                                         : MD4_SUM_LENGTH) == 0;
  93         }
  94
  95         if (size_only)
  96                 return 1;
  97
  98         if (ignore_times)
  99                 return 0;
 100
 101         return cmp_modtime(st->st_mtime, file->modtime) == 0;
 102 }
 103
 104
 105 /*
 106  * NULL sum_struct means we have no checksums
 107  */
 108 void write_sum_head(int f, struct sum_struct *sum)
 109 {
 110         static struct sum_struct null_sum;
 111
 112         if (sum == NULL)
 113                 sum = &null_sum;
 114
 115         write_int(f, sum->count);
 116         write_int(f, sum->blength);
 117         if (protocol_version >= 27)
 118                 write_int(f, sum->s2length);
 119         write_int(f, sum->remainder);
 120 }
 121
 122 /*
 123  * set (initialize) the size entries in the per-file sum_struct
 124  * calculating dynamic block and checksum sizes.
 125  *
 126  * This is only called from generate_and_send_sums() but is a separate
 127  * function to encapsulate the logic.
 128  *
 129  * The block size is a rounded square root of file length.
 130  *
 131  * The checksum size is determined according to:
 132  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 133  * provided by Donovan Baarda which gives a probability of rsync
 134  * algorithm corrupting data and falling back using the whole md4
 135  * checksums.
 136  *
 137  * This might be made one of several selectable heuristics.
 138  */
 139
 140 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 141 {
 142         unsigned int blength;
 143         int s2length;
 144         uint32 c;
 145         uint64 l;
 146
 147         if (block_size) {
 148                 blength = block_size;
 149         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 150                 blength = BLOCK_SIZE;
 151         } else {
 152                 l = len;
 153                 c = 1;
 154                 while (l >>= 2) {
 155                         c <<= 1;
 156                 }
 157                 blength = 0;
 158                 do {
 159                         blength |= c;
 160                         if (len < (uint64)blength * blength)
 161                                 blength &= ~c;
 162                         c >>= 1;
 163                 } while (c >= 8);       /* round to multiple of 8 */
 164                 blength = MAX(blength, BLOCK_SIZE);
 165                 blength = MIN(blength, MAX_MAP_SIZE);
 166         }
 167
 168         if (protocol_version < 27) {
 169                 s2length = csum_length;
 170         } else if (csum_length == SUM_LENGTH) {
 171                 s2length = SUM_LENGTH;
 172         } else {
 173                 int b = BLOCKSUM_BIAS;
 174                 l = len;
 175                 while (l >>= 1) {
 176                         b += 2;
 177                 }
 178                 c = blength;
 179                 while (c >>= 1 && b) {
 180                         b--;
 181                 }
 182                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 183                                                   * subtract rollsum,
 184                                                   * round up
 185                                                   *    --optimize in compiler--
 186                                                   */
 187                 s2length = MAX(s2length, csum_length);
 188                 s2length = MIN(s2length, SUM_LENGTH);
 189         }
 190
 191         sum->flength    = len;
 192         sum->blength    = blength;
 193         sum->s2length   = s2length;
 194         sum->count      = (len + (blength - 1)) / blength;
 195         sum->remainder  = (len % blength);
 196
 197         if (sum->count && verbose > 2) {
 198                 rprintf(FINFO, "count=%.0f rem=%u blength=%u s2length=%d flength=%.0f\n",
 199                         (double)sum->count, sum->remainder, sum->blength,
 200                         sum->s2length, (double)sum->flength);
 201         }
 202 }
 203
 204
 205 /*
 206  * Generate and send a stream of signatures/checksums that describe a buffer
 207  *
 208  * Generate approximately one checksum every block_len bytes.
 209  */
 210 static void generate_and_send_sums(int fd, OFF_T len, int f_out)
 211 {
 212         size_t i;
 213         struct map_struct *mapbuf;
 214         struct sum_struct sum;
 215         OFF_T offset = 0;
 216
 217         sum_sizes_sqroot(&sum, len);
 218
 219         if (len > 0)
 220                 mapbuf = map_file(fd, len, sum.blength);
 221         else
 222                 mapbuf = NULL;
 223
 224         write_sum_head(f_out, &sum);
 225
 226         for (i = 0; i < sum.count; i++) {
 227                 unsigned int n1 = MIN(len, sum.blength);
 228                 char *map = map_ptr(mapbuf, offset, n1);
 229                 uint32 sum1 = get_checksum1(map, n1);
 230                 char sum2[SUM_LENGTH];
 231
 232                 get_checksum2(map, n1, sum2);
 233
 234                 if (verbose > 3) {
 235                         rprintf(FINFO,
 236                                 "chunk[%.0f] offset=%.0f len=%u sum1=%08lx\n",
 237                                 (double)i, (double)offset, n1,
 238                                 (unsigned long)sum1);
 239                 }
 240                 write_int(f_out, sum1);
 241                 write_buf(f_out, sum2, sum.s2length);
 242                 len -= n1;
 243                 offset += n1;
 244         }
 245
 246         if (mapbuf)
 247                 unmap_file(mapbuf);
 248 }
 249
 250
 251
 252 /*
 253  * Acts on file number @p i from @p flist, whose name is @p fname.
 254  *
 255  * First fixes up permissions, then generates checksums for the file.
 256  *
 257  * @note This comment was added later by mbp who was trying to work it
 258  * out.  It might be wrong.
 259  */
 260 static void recv_generator(char *fname, struct file_struct *file, int i,
 261                            int f_out)
 262 {
 263         int fd;
 264         STRUCT_STAT st;
 265         int statret;
 266         char *fnamecmp;
 267         char fnamecmpbuf[MAXPATHLEN];
 268
 269         if (list_only)
 270                 return;
 271
 272         if (verbose > 2)
 273                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 274
 275         if (server_exclude_list.head
 276             && check_exclude(&server_exclude_list, fname,
 277                              S_ISDIR(file->mode)) < 0) {
 278                 if (verbose) {
 279                         rprintf(FINFO, "skipping server-excluded file \"%s\"\n",
 280                                 fname);
 281                 }
 282                 return;
 283         }
 284
 285         statret = link_stat(fname, &st, keep_dirlinks && S_ISDIR(file->mode));
 286
 287         if (only_existing && statret == -1 && errno == ENOENT) {
 288                 /* we only want to update existing files */
 289                 if (verbose > 1)
 290                         rprintf(FINFO, "not creating new file \"%s\"\n", fname);
 291                 return;
 292         }
 293
 294         if (statret == 0 &&
 295             !preserve_perms &&
 296             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 297                 /* if the file exists already and we aren't perserving
 298                  * permissions then act as though the remote end sent
 299                  * us the file permissions we already have */
 300                 file->mode = (file->mode & ~CHMOD_BITS)
 301                            | (st.st_mode & CHMOD_BITS);
 302         }
 303
 304         if (S_ISDIR(file->mode)) {
 305                 /* The file to be received is a directory, so we need
 306                  * to prepare appropriately.  If there is already a
 307                  * file of that name and it is *not* a directory, then
 308                  * we need to delete it.  If it doesn't exist, then
 309                  * recursively create it. */
 310
 311                 if (dry_run)
 312                         return; /* TODO: causes inaccuracies -- fix */
 313                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 314                         if (robust_unlink(fname) != 0) {
 315                                 rsyserr(FERROR, errno,
 316                                         "recv_generator: unlink %s to make room for directory",
 317                                         full_fname(fname));
 318                                 return;
 319                         }
 320                         statret = -1;
 321                 }
 322                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 323                         if (!(relative_paths && errno == ENOENT
 324                             && create_directory_path(fname, orig_umask) == 0
 325                             && do_mkdir(fname, file->mode) == 0)) {
 326                                 rsyserr(FERROR, errno,
 327                                         "recv_generator: mkdir %s failed",
 328                                         full_fname(fname));
 329                         }
 330                 }
 331                 /* f_out is set to -1 when doing final directory-permission
 332                  * and modification-time repair. */
 333                 if (set_perms(fname, file, statret ? NULL : &st, 0)
 334                     && verbose && f_out != -1)
 335                         rprintf(FINFO,"%s/\n",fname);
 336                 return;
 337         }
 338
 339         if (preserve_links && S_ISLNK(file->mode)) {
 340 #if SUPPORT_LINKS
 341                 char lnk[MAXPATHLEN];
 342                 int l;
 343
 344                 if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
 345                         if (verbose) {
 346                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 347                                         full_fname(fname), file->u.link);
 348                         }
 349                         return;
 350                 }
 351                 if (statret == 0) {
 352                         l = readlink(fname,lnk,MAXPATHLEN-1);
 353                         if (l > 0) {
 354                                 lnk[l] = 0;
 355                                 /* A link already pointing to the
 356                                  * right place -- no further action
 357                                  * required. */
 358                                 if (strcmp(lnk,file->u.link) == 0) {
 359                                         set_perms(fname, file, &st,
 360                                                   PERMS_REPORT);
 361                                         return;
 362                                 }
 363                         }
 364                         /* Not a symlink, so delete whatever's
 365                          * already there and put a new symlink
 366                          * in place. */
 367                         delete_file(fname);
 368                 }
 369                 if (do_symlink(file->u.link,fname) != 0) {
 370                         rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
 371                                 full_fname(fname), file->u.link);
 372                 } else {
 373                         set_perms(fname,file,NULL,0);
 374                         if (verbose) {
 375                                 rprintf(FINFO,"%s -> %s\n", fname,file->u.link);
 376                         }
 377                 }
 378 #endif
 379                 return;
 380         }
 381
 382 #ifdef HAVE_MKNOD
 383         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 384                 if (statret != 0 ||
 385                     st.st_mode != file->mode ||
 386                     st.st_rdev != file->u.rdev) {
 387                         delete_file(fname);
 388                         if (verbose > 2) {
 389                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 390                                         fname,(int)file->mode,(int)file->u.rdev);
 391                         }
 392                         if (do_mknod(fname,file->mode,file->u.rdev) != 0) {
 393                                 rsyserr(FERROR, errno, "mknod %s failed",
 394                                         full_fname(fname));
 395                         } else {
 396                                 set_perms(fname,file,NULL,0);
 397                                 if (verbose)
 398                                         rprintf(FINFO,"%s\n",fname);
 399                         }
 400                 } else {
 401                         set_perms(fname, file, &st, PERMS_REPORT);
 402                 }
 403                 return;
 404         }
 405 #endif
 406
 407         if (preserve_hard_links && hard_link_check(file, HL_CHECK_MASTER))
 408                 return;
 409
 410         if (!S_ISREG(file->mode)) {
 411                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 412                 return;
 413         }
 414
 415         fnamecmp = fname;
 416
 417         if (statret == -1 && compare_dest != NULL) {
 418                 /* try the file at compare_dest instead */
 419                 int saveerrno = errno;
 420                 pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, compare_dest, fname);
 421                 statret = link_stat(fnamecmpbuf, &st, 0);
 422                 if (!S_ISREG(st.st_mode))
 423                         statret = -1;
 424                 if (statret == -1)
 425                         errno = saveerrno;
 426 #if HAVE_LINK
 427                 else if (link_dest && !dry_run) {
 428                         if (do_link(fnamecmpbuf, fname) != 0) {
 429                                 if (verbose > 0) {
 430                                         rsyserr(FINFO, errno, "link %s => %s",
 431                                                 fnamecmpbuf, fname);
 432                                 }
 433                         }
 434                         fnamecmp = fnamecmpbuf;
 435                 }
 436 #endif
 437                 else
 438                         fnamecmp = fnamecmpbuf;
 439         }
 440
 441         if (statret == -1) {
 442                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 443                         return;
 444                 if (errno == ENOENT) {
 445                         write_int(f_out,i);
 446                         if (!dry_run && !read_batch)
 447                                 write_sum_head(f_out, NULL);
 448                 } else if (verbose > 1) {
 449                         rsyserr(FERROR, errno,
 450                                 "recv_generator: failed to open %s",
 451                                 full_fname(fname));
 452                 }
 453                 return;
 454         }
 455
 456         if (!S_ISREG(st.st_mode)) {
 457                 if (delete_file(fname) != 0) {
 458                         return;
 459                 }
 460
 461                 /* now pretend the file didn't exist */
 462                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 463                         return;
 464                 write_int(f_out,i);
 465                 if (!dry_run && !read_batch)
 466                         write_sum_head(f_out, NULL);
 467                 return;
 468         }
 469
 470         if (opt_ignore_existing && fnamecmp == fname) {
 471                 if (verbose > 1)
 472                         rprintf(FINFO,"%s exists\n",fname);
 473                 return;
 474         }
 475
 476         if (update_only && fnamecmp == fname
 477             && cmp_modtime(st.st_mtime, file->modtime) > 0) {
 478                 if (verbose > 1)
 479                         rprintf(FINFO,"%s is newer\n",fname);
 480                 return;
 481         }
 482
 483         if (skip_file(fname, file, &st)) {
 484                 if (fnamecmp == fname)
 485                         set_perms(fname, file, &st, PERMS_REPORT);
 486                 return;
 487         }
 488
 489         if (dry_run || read_batch) {
 490                 write_int(f_out,i);
 491                 return;
 492         }
 493
 494         if (whole_file > 0) {
 495                 write_int(f_out,i);
 496                 write_sum_head(f_out, NULL);
 497                 return;
 498         }
 499
 500         /* open the file */
 501         fd = do_open(fnamecmp, O_RDONLY, 0);
 502
 503         if (fd == -1) {
 504                 rsyserr(FERROR, errno, "failed to open %s, continuing",
 505                         full_fname(fnamecmp));
 506                 /* pretend the file didn't exist */
 507                 if (preserve_hard_links && hard_link_check(file, HL_SKIP))
 508                         return;
 509                 write_int(f_out,i);
 510                 write_sum_head(f_out, NULL);
 511                 return;
 512         }
 513
 514         if (verbose > 3) {
 515                 rprintf(FINFO,"gen mapped %s of size %.0f\n", fnamecmp,
 516                         (double)st.st_size);
 517         }
 518
 519         if (verbose > 2)
 520                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 521
 522         write_int(f_out,i);
 523         generate_and_send_sums(fd, st.st_size, f_out);
 524
 525         close(fd);
 526 }
 527
 528
 529 void generate_files(int f_out, struct file_list *flist, char *local_name)
 530 {
 531         int i;
 532         int phase = 0;
 533         char fbuf[MAXPATHLEN];
 534
 535         if (verbose > 2) {
 536                 rprintf(FINFO, "generator starting pid=%ld count=%d\n",
 537                         (long)getpid(), flist->count);
 538         }
 539
 540         if (verbose >= 2) {
 541                 rprintf(FINFO,
 542                         whole_file > 0
 543                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 544                         : "delta transmission enabled\n");
 545         }
 546
 547         /* we expect to just sit around now, so don't exit on a
 548            timeout. If we really get a timeout then the other process should
 549            exit */
 550         io_timeout = 0;
 551
 552         for (i = 0; i < flist->count; i++) {
 553                 struct file_struct *file = flist->files[i];
 554                 struct file_struct copy;
 555
 556                 if (!file->basename)
 557                         continue;
 558                 /* we need to ensure that any directories we create have writeable
 559                    permissions initially so that we can create the files within
 560                    them. This is then fixed after the files are transferred */
 561                 if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)) {
 562                         copy = *file;
 563                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 564                          * handling of permissions is strange? */
 565                         copy.mode |= S_IWUSR; /* user write */
 566                         file = &copy;
 567                 }
 568
 569                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 570                                file, i, f_out);
 571         }
 572
 573         phase++;
 574         csum_length = SUM_LENGTH;
 575         ignore_times = 1;
 576
 577         if (verbose > 2)
 578                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 579
 580         write_int(f_out, -1);
 581
 582         /* files can cycle through the system more than once
 583          * to catch initial checksum errors */
 584         while ((i = get_redo_num()) != -1) {
 585                 struct file_struct *file = flist->files[i];
 586                 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
 587                                file, i, f_out);
 588         }
 589
 590         phase++;
 591         if (verbose > 2)
 592                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 593
 594         write_int(f_out, -1);
 595
 596         if (preserve_hard_links)
 597                 do_hard_links();
 598
 599         /* now we need to fix any directory permissions that were
 600          * modified during the transfer */
 601         for (i = 0; i < flist->count; i++) {
 602                 struct file_struct *file = flist->files[i];
 603                 if (!file->basename || !S_ISDIR(file->mode))
 604                         continue;
 605                 recv_generator(local_name ? local_name : f_name(file),
 606                                file, i, -1);
 607         }
 608
 609         if (verbose > 2)
 610                 rprintf(FINFO,"generate_files finished\n");
 611 }