generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int csum_length;
  36 extern int ignore_times;
  37 extern int size_only;
  38 extern int io_timeout;
  39 extern int protocol_version;
  40 extern int always_checksum;
  41 extern char *compare_dest;
  42 extern int link_dest;
  43
  44
  45 /* choose whether to skip a particular file */
  46 static int skip_file(char *fname,
  47                      struct file_struct *file, STRUCT_STAT *st)
  48 {
  49         if (st->st_size != file->length) {
  50                 return 0;
  51         }
  52         if (link_dest) {
  53                 extern int preserve_perms;
  54                 extern int preserve_uid;
  55                 extern int preserve_gid;
  56
  57                 if (preserve_perms
  58                     && (st->st_mode & ~_S_IFMT) != (file->mode & ~_S_IFMT))
  59                         return 0;
  60
  61                 if (preserve_uid && st->st_uid != file->uid)
  62                         return 0;
  63
  64                 if (preserve_gid && st->st_gid != file->gid)
  65                         return 0;
  66         }
  67
  68         /* if always checksum is set then we use the checksum instead
  69            of the file time to determine whether to sync */
  70         if (always_checksum && S_ISREG(st->st_mode)) {
  71                 char sum[MD4_SUM_LENGTH];
  72                 char fnamecmpdest[MAXPATHLEN];
  73
  74                 if (compare_dest != NULL) {
  75                         if (access(fname, 0) != 0) {
  76                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  77                                          compare_dest,fname);
  78                                 fname = fnamecmpdest;
  79                         }
  80                 }
  81                 file_checksum(fname,sum,st->st_size);
  82                 if (protocol_version < 21) {
  83                         return (memcmp(sum,file->sum,2) == 0);
  84                 } else {
  85                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  86                 }
  87         }
  88
  89         if (size_only) {
  90                 return 1;
  91         }
  92
  93         if (ignore_times) {
  94                 return 0;
  95         }
  96
  97         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  98 }
  99
 100
 101 /*
 102  *      NULL sum_struct means we have no checksums
 103  */
 104
 105 void write_sum_head(int f, struct sum_struct *sum)
 106 {
 107         static struct sum_struct null_sum;
 108
 109         if (sum == (struct sum_struct *)NULL)
 110                 sum = &null_sum;
 111
 112         write_int(f, sum->count);
 113         write_int(f, sum->blength);
 114         if (protocol_version >= 27)
 115                 write_int(f, sum->s2length);
 116         write_int(f, sum->remainder);
 117 }
 118
 119 /*
 120  * set (initialize) the size entries in the per-file sum_struct
 121  * calulating dynamic block ans checksum sizes.
 122  *
 123  * This is only called from generate_and_send_sums() but is a seperate
 124  * function to encapsulate the logic.
 125  *
 126  * The block size is a rounded square root of file length.
 127  *
 128  * The checksum size is determined according to:
 129  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 130  * provided by Donovan Baarda which gives a probability of rsync
 131  * algorithm corrupting data and falling back using the whole md4
 132  * checksums.
 133  *
 134  * This might be made one of several selectable heuristics.
 135  */
 136
 137 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 138 {
 139         extern int block_size;
 140         int blength, s2length, b;
 141         uint32 c;
 142         uint64 l;
 143
 144         if (block_size) {
 145                 blength = block_size;
 146         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 147                 blength = BLOCK_SIZE;
 148         } else {
 149                 l = len;
 150                 c = 1;
 151                 while (l >>= 2) {
 152                         c <<= 1;
 153                 }
 154                 blength = 0;
 155                 do {
 156                         blength |= c;
 157                         if (len < (uint64)blength * blength)
 158                                 blength &= ~c;
 159                         c >>= 1;
 160                 } while (c >= 8);       /* round to multiple of 8 */
 161                 blength = MAX(blength, BLOCK_SIZE);
 162         }
 163
 164         if (protocol_version < 27) {
 165                 s2length = csum_length;
 166         } else if (csum_length == SUM_LENGTH) {
 167                 s2length = SUM_LENGTH;
 168         } else {
 169                 b = BLOCKSUM_BIAS;
 170                 l = len;
 171                 while (l >>= 1) {
 172                         b += 2;
 173                 }
 174                 c = blength;
 175                 while (c >>= 1 && b) {
 176                         b--;
 177                 }
 178                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 179                                                   * subtract rollsum,
 180                                                   * round up
 181                                                   *    --optimize in compiler--
 182                                                   */
 183                 s2length = MAX(s2length, csum_length);
 184                 s2length = MIN(s2length, SUM_LENGTH);
 185         }
 186
 187         sum->flength    = len;
 188         sum->blength    = blength;
 189         sum->s2length   = s2length;
 190         sum->count      = (len + (blength - 1)) / blength;
 191         sum->remainder  = (len % blength);
 192
 193         if (sum->count && verbose > 2) {
 194                 rprintf(FINFO, "count=%ld rem=%ld blength=%ld s2length=%ld flength=%.0f\n",
 195                         (long) sum->count, (long) sum->remainder,
 196                         (long) sum->blength, (long) sum->s2length,
 197                         (double) sum->flength);
 198         }
 199 }
 200
 201 /**
 202  * Perhaps we want to just send an empty checksum set for this file,
 203  * which will force the whole thing to be literally transferred.
 204  *
 205  * When do we do this?  If the user's explicitly said they
 206  * want the whole thing, or if { they haven't explicitly
 207  * requested a delta, and it's local but not batch mode.}
 208  *
 209  * Whew. */
 210 static BOOL disable_deltas_p(void)
 211 {
 212         extern int whole_file;
 213         extern int local_server;
 214         extern int write_batch;
 215
 216         if (whole_file > 0)
 217                 return True;
 218         if (whole_file == 0 || write_batch)
 219                 return False;
 220         return local_server;
 221 }
 222
 223
 224 /*
 225  * Generate and send a stream of signatures/checksums that describe a buffer
 226  *
 227  * Generate approximately one checksum every block_len bytes.
 228  */
 229 static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out)
 230 {
 231         size_t i;
 232         struct sum_struct sum;
 233         OFF_T offset = 0;
 234
 235         sum_sizes_sqroot(&sum, len);
 236
 237         write_sum_head(f_out, &sum);
 238
 239         for (i = 0; i < sum.count; i++) {
 240                 int n1 = MIN(len, sum.blength);
 241                 char *map = map_ptr(buf, offset, n1);
 242                 uint32 sum1 = get_checksum1(map, n1);
 243                 char sum2[SUM_LENGTH];
 244
 245                 get_checksum2(map, n1, sum2);
 246
 247                 if (verbose > 3) {
 248                         rprintf(FINFO,
 249                                 "chunk[%ld] offset=%.0f len=%d sum1=%08lx\n",
 250                                 (long)i,(double)offset,n1,(unsigned long)sum1);
 251                 }
 252                 write_int(f_out, sum1);
 253                 write_buf(f_out, sum2, sum.s2length);
 254                 len -= n1;
 255                 offset += n1;
 256         }
 257 }
 258
 259
 260
 261 /**
 262  * Acts on file number @p i from @p flist, whose name is @p fname.
 263  *
 264  * First fixes up permissions, then generates checksums for the file.
 265  *
 266  * @note This comment was added later by mbp who was trying to work it
 267  * out.  It might be wrong.
 268  **/
 269 void recv_generator(char *fname, struct file_list *flist, int i, int f_out)
 270 {
 271         int fd;
 272         STRUCT_STAT st;
 273         struct map_struct *buf;
 274         int statret;
 275         struct file_struct *file = flist->files[i];
 276         char *fnamecmp;
 277         char fnamecmpbuf[MAXPATHLEN];
 278         extern char *compare_dest;
 279         extern int list_only;
 280         extern int preserve_perms;
 281         extern int only_existing;
 282         extern int orig_umask;
 283
 284         if (list_only) return;
 285
 286         if (verbose > 2)
 287                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 288
 289         statret = link_stat(fname,&st);
 290
 291         if (only_existing && statret == -1 && errno == ENOENT) {
 292                 /* we only want to update existing files */
 293                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 294                 return;
 295         }
 296
 297         if (statret == 0 &&
 298             !preserve_perms &&
 299             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 300                 /* if the file exists already and we aren't perserving
 301                  * permissions then act as though the remote end sent
 302                  * us the file permissions we already have */
 303                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 304         }
 305
 306         if (S_ISDIR(file->mode)) {
 307                 /* The file to be received is a directory, so we need
 308                  * to prepare appropriately.  If there is already a
 309                  * file of that name and it is *not* a directory, then
 310                  * we need to delete it.  If it doesn't exist, then
 311                  * recursively create it. */
 312
 313                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 314                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 315                         if (robust_unlink(fname) != 0) {
 316                                 rprintf(FERROR,
 317                                         "recv_generator: unlink %s to make room for directory: %s\n",
 318                                         full_fname(fname), strerror(errno));
 319                                 return;
 320                         }
 321                         statret = -1;
 322                 }
 323                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 324                         if (!(relative_paths && errno==ENOENT &&
 325                               create_directory_path(fname, orig_umask)==0 &&
 326                               do_mkdir(fname,file->mode)==0)) {
 327                                 rprintf(FERROR, "recv_generator: mkdir %s failed: %s\n",
 328                                         full_fname(fname), strerror(errno));
 329                         }
 330                 }
 331                 /* f_out is set to -1 when doing final directory
 332                    permission and modification time repair */
 333                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 334                         rprintf(FINFO,"%s/\n",fname);
 335                 return;
 336         }
 337
 338         if (preserve_links && S_ISLNK(file->mode)) {
 339 #if SUPPORT_LINKS
 340                 char lnk[MAXPATHLEN];
 341                 int l;
 342                 extern int safe_symlinks;
 343
 344                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 345                         if (verbose) {
 346                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 347                                         full_fname(fname), file->link);
 348                         }
 349                         return;
 350                 }
 351                 if (statret == 0) {
 352                         l = readlink(fname,lnk,MAXPATHLEN-1);
 353                         if (l > 0) {
 354                                 lnk[l] = 0;
 355                                 /* A link already pointing to the
 356                                  * right place -- no further action
 357                                  * required. */
 358                                 if (strcmp(lnk,file->link) == 0) {
 359                                         set_perms(fname,file,&st,1);
 360                                         return;
 361                                 }
 362                         }
 363                         /* Not a symlink, so delete whatever's
 364                          * already there and put a new symlink
 365                          * in place. */
 366                         delete_file(fname);
 367                 }
 368                 if (do_symlink(file->link,fname) != 0) {
 369                         rprintf(FERROR, "symlink %s -> \"%s\" failed: %s\n",
 370                                 full_fname(fname), file->link, strerror(errno));
 371                 } else {
 372                         set_perms(fname,file,NULL,0);
 373                         if (verbose) {
 374                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 375                         }
 376                 }
 377 #endif
 378                 return;
 379         }
 380
 381 #ifdef HAVE_MKNOD
 382         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 383                 if (statret != 0 ||
 384                     st.st_mode != file->mode ||
 385                     (DEV64_T)st.st_rdev != file->rdev) {
 386                         delete_file(fname);
 387                         if (verbose > 2)
 388                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 389                                         fname,(int)file->mode,(int)file->rdev);
 390                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 391                                 rprintf(FERROR, "mknod %s failed: %s\n",
 392                                         full_fname(fname), strerror(errno));
 393                         } else {
 394                                 set_perms(fname,file,NULL,0);
 395                                 if (verbose)
 396                                         rprintf(FINFO,"%s\n",fname);
 397                         }
 398                 } else {
 399                         set_perms(fname,file,&st,1);
 400                 }
 401                 return;
 402         }
 403 #endif
 404
 405         if (preserve_hard_links && check_hard_link(file)) {
 406                 if (verbose > 1) {
 407                         rprintf(FINFO, "recv_generator: \"%s\" is a hard link\n",
 408                                 f_name(file));
 409                 }
 410                 return;
 411         }
 412
 413         if (!S_ISREG(file->mode)) {
 414                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 415                 return;
 416         }
 417
 418         fnamecmp = fname;
 419
 420         if ((statret == -1) && (compare_dest != NULL)) {
 421                 /* try the file at compare_dest instead */
 422                 int saveerrno = errno;
 423                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 424                 statret = link_stat(fnamecmpbuf,&st);
 425                 if (!S_ISREG(st.st_mode))
 426                         statret = -1;
 427                 if (statret == -1)
 428                         errno = saveerrno;
 429 #if HAVE_LINK
 430                 else if (link_dest && !dry_run) {
 431                         if (do_link(fnamecmpbuf, fname) != 0) {
 432                                 if (verbose > 0) {
 433                                         rprintf(FINFO,"link %s => %s : %s\n",
 434                                                 fnamecmpbuf, fname,
 435                                                 strerror(errno));
 436                                 }
 437                         }
 438                         fnamecmp = fnamecmpbuf;
 439                 }
 440 #endif
 441                 else
 442                         fnamecmp = fnamecmpbuf;
 443         }
 444
 445         if (statret == -1) {
 446                 if (errno == ENOENT) {
 447                         write_int(f_out,i);
 448                         if (!dry_run) write_sum_head(f_out, NULL);
 449                 } else if (verbose > 1) {
 450                         rprintf(FERROR,
 451                                 "recv_generator: failed to open %s: %s\n",
 452                                 full_fname(fname), strerror(errno));
 453                 }
 454                 return;
 455         }
 456
 457         if (!S_ISREG(st.st_mode)) {
 458                 if (delete_file(fname) != 0) {
 459                         return;
 460                 }
 461
 462                 /* now pretend the file didn't exist */
 463                 write_int(f_out,i);
 464                 if (!dry_run) write_sum_head(f_out, NULL);
 465                 return;
 466         }
 467
 468         if (opt_ignore_existing && fnamecmp == fname) {
 469                 if (verbose > 1)
 470                         rprintf(FINFO,"%s exists\n",fname);
 471                 return;
 472         }
 473
 474         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 475                 if (verbose > 1)
 476                         rprintf(FINFO,"%s is newer\n",fname);
 477                 return;
 478         }
 479
 480         if (skip_file(fname, file, &st)) {
 481                 if (fnamecmp == fname)
 482                         set_perms(fname,file,&st,1);
 483                 return;
 484         }
 485
 486         if (dry_run) {
 487                 write_int(f_out,i);
 488                 return;
 489         }
 490
 491         if (disable_deltas_p()) {
 492                 write_int(f_out,i);
 493                 write_sum_head(f_out, NULL);
 494                 return;
 495         }
 496
 497         /* open the file */
 498         fd = do_open(fnamecmp, O_RDONLY, 0);
 499
 500         if (fd == -1) {
 501                 rprintf(FERROR, "failed to open %s, continuing: %s\n",
 502                         full_fname(fnamecmp), strerror(errno));
 503                 /* pretend the file didn't exist */
 504                 write_int(f_out,i);
 505                 write_sum_head(f_out, NULL);
 506                 return;
 507         }
 508
 509         if (st.st_size > 0) {
 510                 buf = map_file(fd,st.st_size);
 511         } else {
 512                 buf = NULL;
 513         }
 514
 515         if (verbose > 3)
 516                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 517
 518         if (verbose > 2)
 519                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 520
 521         write_int(f_out,i);
 522         generate_and_send_sums(buf, st.st_size, f_out);
 523
 524         close(fd);
 525         if (buf) unmap_file(buf);
 526 }
 527
 528
 529
 530 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 531 {
 532         int i;
 533         int phase=0;
 534         char buf[MAXPATHLEN];
 535
 536         if (verbose > 2)
 537                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 538                         (int)getpid(),flist->count);
 539
 540         if (verbose >= 2) {
 541                 rprintf(FINFO,
 542                         disable_deltas_p()
 543                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 544                         : "delta transmission enabled\n");
 545         }
 546
 547         /* we expect to just sit around now, so don't exit on a
 548            timeout. If we really get a timeout then the other process should
 549            exit */
 550         io_timeout = 0;
 551
 552         for (i = 0; i < flist->count; i++) {
 553                 struct file_struct *file = flist->files[i];
 554                 mode_t saved_mode = file->mode;
 555                 if (!file->basename) continue;
 556
 557                 /* we need to ensure that any directories we create have writeable
 558                    permissions initially so that we can create the files within
 559                    them. This is then fixed after the files are transferred */
 560                 if (!am_root && S_ISDIR(file->mode)) {
 561                         file->mode |= S_IWUSR; /* user write */
 562                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 563                          * handling of permissions is strange? */
 564                 }
 565
 566                 recv_generator(local_name? local_name
 567                              : f_name_to(file, buf, sizeof buf), flist, i, f);
 568
 569                 file->mode = saved_mode;
 570         }
 571
 572         phase++;
 573         csum_length = SUM_LENGTH;
 574         ignore_times=1;
 575
 576         if (verbose > 2)
 577                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 578
 579         write_int(f,-1);
 580
 581         /* files can cycle through the system more than once
 582          * to catch initial checksum errors */
 583         for (i = read_int(f_recv); i != -1; i = read_int(f_recv)) {
 584                 struct file_struct *file = flist->files[i];
 585                 recv_generator(local_name? local_name
 586                              : f_name_to(file, buf, sizeof buf), flist, i, f);
 587         }
 588
 589         phase++;
 590         if (verbose > 2)
 591                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 592
 593         write_int(f,-1);
 594 }