generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int csum_length;
  36 extern int ignore_times;
  37 extern int size_only;
  38 extern int io_timeout;
  39 extern int remote_version;
  40 extern int always_checksum;
  41 extern int modify_window;
  42 extern char *compare_dest;
  43 extern int link_dest;
  44
  45
  46 /* choose whether to skip a particular file */
  47 static int skip_file(char *fname,
  48                      struct file_struct *file, STRUCT_STAT *st)
  49 {
  50         if (st->st_size != file->length) {
  51                 return 0;
  52         }
  53         if (link_dest) {
  54                 extern int preserve_perms;
  55                 extern int preserve_uid;
  56                 extern int preserve_gid;
  57
  58                 if(preserve_perms
  59                     && (st->st_mode & ~_S_IFMT) !=  (file->mode & ~_S_IFMT))
  60                         return 0;
  61
  62                 if (preserve_uid && st->st_uid != file->uid)
  63                         return 0;
  64
  65                 if (preserve_gid && st->st_gid != file->gid)
  66                         return 0;
  67         }
  68
  69         /* if always checksum is set then we use the checksum instead
  70            of the file time to determine whether to sync */
  71         if (always_checksum && S_ISREG(st->st_mode)) {
  72                 char sum[MD4_SUM_LENGTH];
  73                 char fnamecmpdest[MAXPATHLEN];
  74
  75                 if (compare_dest != NULL) {
  76                         if (access(fname, 0) != 0) {
  77                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  78                                          compare_dest,fname);
  79                                 fname = fnamecmpdest;
  80                         }
  81                 }
  82                 file_checksum(fname,sum,st->st_size);
  83                 if (remote_version < 21) {
  84                         return (memcmp(sum,file->sum,2) == 0);
  85                 } else {
  86                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  87                 }
  88         }
  89
  90         if (size_only) {
  91                 return 1;
  92         }
  93
  94         if (ignore_times) {
  95                 return 0;
  96         }
  97
  98         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  99 }
 100
 101
 102 /*
 103  *      NULL sum_struct means we have no checksums
 104  */
 105
 106 void write_sum_head(int f, struct sum_struct *sum)
 107 {
 108         static struct sum_struct null_sum;
 109
 110         if (sum == (struct sum_struct *)NULL)
 111                 sum = &null_sum;
 112
 113         write_int(f, sum->count);
 114         write_int(f, sum->blength);
 115         if (remote_version >= 27)
 116                 write_int(f, sum->s2length);
 117         write_int(f, sum->remainder);
 118 }
 119
 120 /*
 121  * set (initialize) the size entries in the per-file sum_struct
 122  * calulating dynamic block ans checksum sizes.
 123  *
 124  * This is only called from generate_and_send_sums() but is a seperate
 125  * function to encapsulate the logic.
 126  *
 127  * The block size is a rounded square root of file length.
 128  *
 129  * The checksum size is determined according to:
 130  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 131  * provided by Donovan Baarda which gives a probability of rsync
 132  * algorithm corrupting data and falling back using the whole md4
 133  * checksums.
 134  *
 135  * This might be made one of several selectable heuristics.
 136  */
 137
 138 static void sum_sizes_sqroot_baarda(struct sum_struct *sum, uint64 len)
 139 {
 140         extern int block_size;
 141         int blength, s2length, b;
 142         uint32 c;
 143         uint64 l;
 144
 145         if (block_size) {
 146                 blength = block_size;
 147         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 148                 blength = BLOCK_SIZE;
 149         } else {
 150                 l = len;
 151                 c = 1;
 152                 while (l >>= 2) {
 153                         c <<= 1;
 154                 }
 155                 blength = 0;
 156                 do {
 157                         blength |= c;
 158                         if (len < (uint64)(blength * blength))
 159                                 blength &= ~c;
 160                         c >>= 1;
 161                 } while (c >= 8);       /* round to multiple of 8 */
 162                 blength = MAX(blength, BLOCK_SIZE);
 163         }
 164
 165         if (remote_version < 27) {
 166                 s2length = csum_length;
 167         } else if (csum_length == SUM_LENGTH) {
 168                 s2length = SUM_LENGTH;
 169         } else {
 170                 b = BLOCKSUM_BIAS;
 171                 l = len;
 172                 while (l >>= 1) {
 173                         b += 2;
 174                 }
 175                 c = blength;
 176                 while (c >>= 1 && b) {
 177                         b--;
 178                 }
 179                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 180                                                   * subtract rollsum,
 181                                                   * round up
 182                                                   *    --optimize in compiler--
 183                                                   */
 184                 s2length = MAX(s2length, csum_length);
 185                 s2length = MIN(s2length, SUM_LENGTH);
 186         }
 187
 188         sum->flength    = len;
 189         sum->blength    = blength;
 190         sum->s2length   = s2length;
 191         sum->count      = (len + (blength - 1)) / blength;
 192         sum->remainder  = (len % blength);
 193
 194         if (sum->count && verbose > 2) {
 195                 rprintf(FINFO, "count=%ld rem=%ld blength=%ld s2length=%ld flength=%.0f\n",
 196                         (long) sum->count, (long) sum->remainder,
 197                         (long) sum->blength, (long) sum->s2length,
 198                         (double) sum->flength);
 199         }
 200 }
 201
 202 /**
 203  * Perhaps we want to just send an empty checksum set for this file,
 204  * which will force the whole thing to be literally transferred.
 205  *
 206  * When do we do this?  If the user's explicitly said they
 207  * want the whole thing, or if { they haven't explicitly
 208  * requested a delta, and it's local but not batch mode.}
 209  *
 210  * Whew. */
 211 static BOOL disable_deltas_p(void)
 212 {
 213         extern int whole_file;
 214         extern int local_server;
 215         extern int write_batch;
 216
 217         if (whole_file > 0)
 218                 return True;
 219         if (whole_file == 0 || write_batch)
 220                 return False;
 221         return local_server;
 222 }
 223
 224
 225 /*
 226  * Generate and send a stream of signatures/checksums that describe a buffer
 227  *
 228  * Generate approximately one checksum every block_len bytes.
 229  */
 230 static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out)
 231 {
 232         size_t i;
 233         struct sum_struct sum;
 234         OFF_T offset = 0;
 235
 236         sum_sizes_sqroot_baarda(&sum, len);
 237
 238         write_sum_head(f_out, &sum);
 239
 240         for (i = 0; i < sum.count; i++) {
 241                 int n1 = MIN(len, sum.blength);
 242                 char *map = map_ptr(buf, offset, n1);
 243                 uint32 sum1 = get_checksum1(map, n1);
 244                 char sum2[SUM_LENGTH];
 245
 246                 get_checksum2(map, n1, sum2);
 247
 248                 if (verbose > 3) {
 249                         rprintf(FINFO,
 250                                 "chunk[%ld] offset=%.0f len=%d sum1=%08lx\n",
 251                                 (long)i,(double)offset,n1,(unsigned long)sum1);
 252                 }
 253                 write_int(f_out, sum1);
 254                 write_buf(f_out, sum2, sum.s2length);
 255                 len -= n1;
 256                 offset += n1;
 257         }
 258 }
 259
 260
 261
 262 /**
 263  * Acts on file number @p i from @p flist, whose name is @p fname.
 264  *
 265  * First fixes up permissions, then generates checksums for the file.
 266  *
 267  * @note This comment was added later by mbp who was trying to work it
 268  * out.  It might be wrong.
 269  **/
 270 void recv_generator(char *fname, struct file_list *flist, int i, int f_out)
 271 {
 272         int fd;
 273         STRUCT_STAT st;
 274         struct map_struct *buf;
 275         int statret;
 276         struct file_struct *file = flist->files[i];
 277         char *fnamecmp;
 278         char fnamecmpbuf[MAXPATHLEN];
 279         extern char *compare_dest;
 280         extern int list_only;
 281         extern int preserve_perms;
 282         extern int only_existing;
 283         extern int orig_umask;
 284
 285         if (list_only) return;
 286
 287         if (verbose > 2)
 288                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 289
 290         statret = link_stat(fname,&st);
 291
 292         if (only_existing && statret == -1 && errno == ENOENT) {
 293                 /* we only want to update existing files */
 294                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 295                 return;
 296         }
 297
 298         if (statret == 0 &&
 299             !preserve_perms &&
 300             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 301                 /* if the file exists already and we aren't perserving
 302                  * permissions then act as though the remote end sent
 303                  * us the file permissions we already have */
 304                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 305         }
 306
 307         if (S_ISDIR(file->mode)) {
 308                 /* The file to be received is a directory, so we need
 309                  * to prepare appropriately.  If there is already a
 310                  * file of that name and it is *not* a directory, then
 311                  * we need to delete it.  If it doesn't exist, then
 312                  * recursively create it. */
 313
 314                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 315                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 316                         if (robust_unlink(fname) != 0) {
 317                                 rprintf(FERROR, RSYNC_NAME
 318                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 319                                         fname,strerror(errno));
 320                                 return;
 321                         }
 322                         statret = -1;
 323                 }
 324                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 325                         if (!(relative_paths && errno==ENOENT &&
 326                               create_directory_path(fname, orig_umask)==0 &&
 327                               do_mkdir(fname,file->mode)==0)) {
 328                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 329                                         fname,strerror(errno));
 330                         }
 331                 }
 332                 /* f_out is set to -1 when doing final directory
 333                    permission and modification time repair */
 334                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 335                         rprintf(FINFO,"%s/\n",fname);
 336                 return;
 337         }
 338
 339         if (preserve_links && S_ISLNK(file->mode)) {
 340 #if SUPPORT_LINKS
 341                 char lnk[MAXPATHLEN];
 342                 int l;
 343                 extern int safe_symlinks;
 344
 345                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 346                         if (verbose) {
 347                                 rprintf(FINFO,"ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 348                                         fname,file->link);
 349                         }
 350                         return;
 351                 }
 352                 if (statret == 0) {
 353                         l = readlink(fname,lnk,MAXPATHLEN-1);
 354                         if (l > 0) {
 355                                 lnk[l] = 0;
 356                                 /* A link already pointing to the
 357                                  * right place -- no further action
 358                                  * required. */
 359                                 if (strcmp(lnk,file->link) == 0) {
 360                                         set_perms(fname,file,&st,1);
 361                                         return;
 362                                 }
 363                         }
 364                         /* Not a symlink, so delete whatever's
 365                          * already there and put a new symlink
 366                          * in place. */
 367                         delete_file(fname);
 368                 }
 369                 if (do_symlink(file->link,fname) != 0) {
 370                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 371                                 fname,file->link,strerror(errno));
 372                 } else {
 373                         set_perms(fname,file,NULL,0);
 374                         if (verbose) {
 375                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 376                         }
 377                 }
 378 #endif
 379                 return;
 380         }
 381
 382 #ifdef HAVE_MKNOD
 383         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 384                 if (statret != 0 ||
 385                     st.st_mode != file->mode ||
 386                     st.st_rdev != file->rdev) {
 387                         delete_file(fname);
 388                         if (verbose > 2)
 389                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 390                                         fname,(int)file->mode,(int)file->rdev);
 391                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 392                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 393                         } else {
 394                                 set_perms(fname,file,NULL,0);
 395                                 if (verbose)
 396                                         rprintf(FINFO,"%s\n",fname);
 397                         }
 398                 } else {
 399                         set_perms(fname,file,&st,1);
 400                 }
 401                 return;
 402         }
 403 #endif
 404
 405         if (preserve_hard_links && check_hard_link(file)) {
 406                 if (verbose > 1)
 407                         rprintf(FINFO, "recv_generator: \"%s\" is a hard link\n",f_name(file));
 408                 return;
 409         }
 410
 411         if (!S_ISREG(file->mode)) {
 412                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 413                 return;
 414         }
 415
 416         fnamecmp = fname;
 417
 418         if ((statret == -1) && (compare_dest != NULL)) {
 419                 /* try the file at compare_dest instead */
 420                 int saveerrno = errno;
 421                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 422                 statret = link_stat(fnamecmpbuf,&st);
 423                 if (!S_ISREG(st.st_mode))
 424                         statret = -1;
 425                 if (statret == -1)
 426                         errno = saveerrno;
 427 #if HAVE_LINK
 428                 else if (link_dest && !dry_run) {
 429                         if (do_link(fnamecmpbuf, fname) != 0) {
 430                                 if (verbose > 0)
 431                                         rprintf(FINFO,"link %s => %s : %s\n",
 432                                                 fnamecmpbuf,
 433                                                 fname,
 434                                                 strerror(errno));
 435                         }
 436                         fnamecmp = fnamecmpbuf;
 437                 }
 438 #endif
 439                 else
 440                         fnamecmp = fnamecmpbuf;
 441         }
 442
 443         if (statret == -1) {
 444                 if (errno == ENOENT) {
 445                         write_int(f_out,i);
 446                         if (!dry_run) write_sum_head(f_out, NULL);
 447                 } else {
 448                         if (verbose > 1)
 449                                 rprintf(FERROR, RSYNC_NAME
 450                                         ": recv_generator failed to open \"%s\": %s\n",
 451                                         fname, strerror(errno));
 452                 }
 453                 return;
 454         }
 455
 456         if (!S_ISREG(st.st_mode)) {
 457                 if (delete_file(fname) != 0) {
 458                         return;
 459                 }
 460
 461                 /* now pretend the file didn't exist */
 462                 write_int(f_out,i);
 463                 if (!dry_run) write_sum_head(f_out, NULL);
 464                 return;
 465         }
 466
 467         if (opt_ignore_existing && fnamecmp == fname) {
 468                 if (verbose > 1)
 469                         rprintf(FINFO,"%s exists\n",fname);
 470                 return;
 471         }
 472
 473         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 474                 if (verbose > 1)
 475                         rprintf(FINFO,"%s is newer\n",fname);
 476                 return;
 477         }
 478
 479         if (skip_file(fname, file, &st)) {
 480                 if (fnamecmp == fname)
 481                         set_perms(fname,file,&st,1);
 482                 return;
 483         }
 484
 485         if (dry_run) {
 486                 write_int(f_out,i);
 487                 return;
 488         }
 489
 490         if (disable_deltas_p()) {
 491                 write_int(f_out,i);
 492                 write_sum_head(f_out, NULL);
 493                 return;
 494         }
 495
 496         /* open the file */
 497         fd = do_open(fnamecmp, O_RDONLY, 0);
 498
 499         if (fd == -1) {
 500                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 501                 /* pretend the file didn't exist */
 502                 write_int(f_out,i);
 503                 write_sum_head(f_out, NULL);
 504                 return;
 505         }
 506
 507         if (st.st_size > 0) {
 508                 buf = map_file(fd,st.st_size);
 509         } else {
 510                 buf = NULL;
 511         }
 512
 513         if (verbose > 3)
 514                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 515
 516         if (verbose > 2)
 517                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 518
 519         write_int(f_out,i);
 520         generate_and_send_sums(buf, st.st_size, f_out);
 521
 522         close(fd);
 523         if (buf) unmap_file(buf);
 524 }
 525
 526
 527
 528 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 529 {
 530         int i;
 531         int phase=0;
 532
 533         if (verbose > 2)
 534                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 535                         (int)getpid(),flist->count);
 536
 537         if (verbose >= 2) {
 538                 rprintf(FINFO,
 539                         disable_deltas_p()
 540                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 541                         : "delta transmission enabled\n");
 542         }
 543
 544         /* we expect to just sit around now, so don't exit on a
 545            timeout. If we really get a timeout then the other process should
 546            exit */
 547         io_timeout = 0;
 548
 549         for (i = 0; i < flist->count; i++) {
 550                 struct file_struct *file = flist->files[i];
 551                 mode_t saved_mode = file->mode;
 552                 if (!file->basename) continue;
 553
 554                 /* we need to ensure that any directories we create have writeable
 555                    permissions initially so that we can create the files within
 556                    them. This is then fixed after the files are transferred */
 557                 if (!am_root && S_ISDIR(file->mode)) {
 558                         file->mode |= S_IWUSR; /* user write */
 559                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 560                          * handling of permissions is strange? */
 561                 }
 562
 563                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 564
 565                 file->mode = saved_mode;
 566         }
 567
 568         phase++;
 569         csum_length = SUM_LENGTH;
 570         ignore_times=1;
 571
 572         if (verbose > 2)
 573                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 574
 575         write_int(f,-1);
 576
 577         /* files can cycle through the system more than once
 578          * to catch initial checksum errors */
 579         for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 580                 struct file_struct *file = flist->files[i];
 581                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 582         }
 583
 584         phase++;
 585         if (verbose > 2)
 586                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 587
 588         write_int(f,-1);
 589 }