generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int preserve_perms;
  34 extern int preserve_uid;
  35 extern int preserve_gid;
  36 extern int update_only;
  37 extern int opt_ignore_existing;
  38 extern int csum_length;
  39 extern int ignore_times;
  40 extern int size_only;
  41 extern int io_timeout;
  42 extern int protocol_version;
  43 extern int always_checksum;
  44 extern char *compare_dest;
  45 extern int link_dest;
  46
  47
  48 /* choose whether to skip a particular file */
  49 static int skip_file(char *fname,
  50                      struct file_struct *file, STRUCT_STAT *st)
  51 {
  52         if (st->st_size != file->length) {
  53                 return 0;
  54         }
  55         if (link_dest) {
  56                 if (preserve_perms
  57                     && (st->st_mode & ~_S_IFMT) != (file->mode & ~_S_IFMT))
  58                         return 0;
  59
  60                 if (am_root && preserve_uid && st->st_uid != file->uid)
  61                         return 0;
  62
  63                 if (preserve_gid && file->gid != GID_NONE
  64                     && st->st_gid != file->gid)
  65                         return 0;
  66         }
  67
  68         /* if always checksum is set then we use the checksum instead
  69            of the file time to determine whether to sync */
  70         if (always_checksum && S_ISREG(st->st_mode)) {
  71                 char sum[MD4_SUM_LENGTH];
  72                 char fnamecmpdest[MAXPATHLEN];
  73
  74                 if (compare_dest != NULL) {
  75                         if (access(fname, 0) != 0) {
  76                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  77                                          compare_dest,fname);
  78                                 fname = fnamecmpdest;
  79                         }
  80                 }
  81                 file_checksum(fname,sum,st->st_size);
  82                 if (protocol_version < 21) {
  83                         return (memcmp(sum,file->sum,2) == 0);
  84                 } else {
  85                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  86                 }
  87         }
  88
  89         if (size_only) {
  90                 return 1;
  91         }
  92
  93         if (ignore_times) {
  94                 return 0;
  95         }
  96
  97         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  98 }
  99
 100
 101 /*
 102  *      NULL sum_struct means we have no checksums
 103  */
 104
 105 void write_sum_head(int f, struct sum_struct *sum)
 106 {
 107         static struct sum_struct null_sum;
 108
 109         if (sum == (struct sum_struct *)NULL)
 110                 sum = &null_sum;
 111
 112         write_int(f, sum->count);
 113         write_int(f, sum->blength);
 114         if (protocol_version >= 27)
 115                 write_int(f, sum->s2length);
 116         write_int(f, sum->remainder);
 117 }
 118
 119 /*
 120  * set (initialize) the size entries in the per-file sum_struct
 121  * calulating dynamic block ans checksum sizes.
 122  *
 123  * This is only called from generate_and_send_sums() but is a seperate
 124  * function to encapsulate the logic.
 125  *
 126  * The block size is a rounded square root of file length.
 127  *
 128  * The checksum size is determined according to:
 129  *     blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
 130  * provided by Donovan Baarda which gives a probability of rsync
 131  * algorithm corrupting data and falling back using the whole md4
 132  * checksums.
 133  *
 134  * This might be made one of several selectable heuristics.
 135  */
 136
 137 static void sum_sizes_sqroot(struct sum_struct *sum, uint64 len)
 138 {
 139         extern int block_size;
 140         int blength, s2length, b;
 141         uint32 c;
 142         uint64 l;
 143
 144         if (block_size) {
 145                 blength = block_size;
 146         } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
 147                 blength = BLOCK_SIZE;
 148         } else {
 149                 l = len;
 150                 c = 1;
 151                 while (l >>= 2) {
 152                         c <<= 1;
 153                 }
 154                 blength = 0;
 155                 do {
 156                         blength |= c;
 157                         if (len < (uint64)blength * blength)
 158                                 blength &= ~c;
 159                         c >>= 1;
 160                 } while (c >= 8);       /* round to multiple of 8 */
 161                 blength = MAX(blength, BLOCK_SIZE);
 162         }
 163
 164         if (protocol_version < 27) {
 165                 s2length = csum_length;
 166         } else if (csum_length == SUM_LENGTH) {
 167                 s2length = SUM_LENGTH;
 168         } else {
 169                 b = BLOCKSUM_BIAS;
 170                 l = len;
 171                 while (l >>= 1) {
 172                         b += 2;
 173                 }
 174                 c = blength;
 175                 while (c >>= 1 && b) {
 176                         b--;
 177                 }
 178                 s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
 179                                                   * subtract rollsum,
 180                                                   * round up
 181                                                   *    --optimize in compiler--
 182                                                   */
 183                 s2length = MAX(s2length, csum_length);
 184                 s2length = MIN(s2length, SUM_LENGTH);
 185         }
 186
 187         sum->flength    = len;
 188         sum->blength    = blength;
 189         sum->s2length   = s2length;
 190         sum->count      = (len + (blength - 1)) / blength;
 191         sum->remainder  = (len % blength);
 192
 193         if (sum->count && verbose > 2) {
 194                 rprintf(FINFO, "count=%ld rem=%ld blength=%ld s2length=%ld flength=%.0f\n",
 195                         (long) sum->count, (long) sum->remainder,
 196                         (long) sum->blength, (long) sum->s2length,
 197                         (double) sum->flength);
 198         }
 199 }
 200
 201 /**
 202  * Perhaps we want to just send an empty checksum set for this file,
 203  * which will force the whole thing to be literally transferred.
 204  *
 205  * When do we do this?  If the user's explicitly said they
 206  * want the whole thing, or if { they haven't explicitly
 207  * requested a delta, and it's local but not batch mode.}
 208  *
 209  * Whew. */
 210 static BOOL disable_deltas_p(void)
 211 {
 212         extern int whole_file;
 213         extern int local_server;
 214         extern int write_batch;
 215
 216         if (whole_file > 0)
 217                 return True;
 218         if (whole_file == 0 || write_batch)
 219                 return False;
 220         return local_server;
 221 }
 222
 223
 224 /*
 225  * Generate and send a stream of signatures/checksums that describe a buffer
 226  *
 227  * Generate approximately one checksum every block_len bytes.
 228  */
 229 static void generate_and_send_sums(struct map_struct *buf, size_t len, int f_out)
 230 {
 231         size_t i;
 232         struct sum_struct sum;
 233         OFF_T offset = 0;
 234
 235         sum_sizes_sqroot(&sum, len);
 236
 237         write_sum_head(f_out, &sum);
 238
 239         for (i = 0; i < sum.count; i++) {
 240                 int n1 = MIN(len, sum.blength);
 241                 char *map = map_ptr(buf, offset, n1);
 242                 uint32 sum1 = get_checksum1(map, n1);
 243                 char sum2[SUM_LENGTH];
 244
 245                 get_checksum2(map, n1, sum2);
 246
 247                 if (verbose > 3) {
 248                         rprintf(FINFO,
 249                                 "chunk[%ld] offset=%.0f len=%d sum1=%08lx\n",
 250                                 (long)i,(double)offset,n1,(unsigned long)sum1);
 251                 }
 252                 write_int(f_out, sum1);
 253                 write_buf(f_out, sum2, sum.s2length);
 254                 len -= n1;
 255                 offset += n1;
 256         }
 257 }
 258
 259
 260
 261 /**
 262  * Acts on file number @p i from @p flist, whose name is @p fname.
 263  *
 264  * First fixes up permissions, then generates checksums for the file.
 265  *
 266  * @note This comment was added later by mbp who was trying to work it
 267  * out.  It might be wrong.
 268  **/
 269 void recv_generator(char *fname, struct file_list *flist, int i, int f_out)
 270 {
 271         int fd;
 272         STRUCT_STAT st;
 273         struct map_struct *mapbuf;
 274         int statret;
 275         struct file_struct *file = flist->files[i];
 276         char *fnamecmp;
 277         char fnamecmpbuf[MAXPATHLEN];
 278         extern char *compare_dest;
 279         extern int list_only;
 280         extern int only_existing;
 281         extern int orig_umask;
 282
 283         if (list_only) return;
 284
 285         if (verbose > 2)
 286                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 287
 288         statret = link_stat(fname,&st);
 289
 290         if (only_existing && statret == -1 && errno == ENOENT) {
 291                 /* we only want to update existing files */
 292                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 293                 return;
 294         }
 295
 296         if (statret == 0 &&
 297             !preserve_perms &&
 298             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 299                 /* if the file exists already and we aren't perserving
 300                  * permissions then act as though the remote end sent
 301                  * us the file permissions we already have */
 302                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 303         }
 304
 305         if (S_ISDIR(file->mode)) {
 306                 /* The file to be received is a directory, so we need
 307                  * to prepare appropriately.  If there is already a
 308                  * file of that name and it is *not* a directory, then
 309                  * we need to delete it.  If it doesn't exist, then
 310                  * recursively create it. */
 311
 312                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 313                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 314                         if (robust_unlink(fname) != 0) {
 315                                 rprintf(FERROR,
 316                                         "recv_generator: unlink %s to make room for directory: %s\n",
 317                                         full_fname(fname), strerror(errno));
 318                                 return;
 319                         }
 320                         statret = -1;
 321                 }
 322                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 323                         if (!(relative_paths && errno==ENOENT &&
 324                               create_directory_path(fname, orig_umask)==0 &&
 325                               do_mkdir(fname,file->mode)==0)) {
 326                                 rprintf(FERROR, "recv_generator: mkdir %s failed: %s\n",
 327                                         full_fname(fname), strerror(errno));
 328                         }
 329                 }
 330                 /* f_out is set to -1 when doing final directory
 331                    permission and modification time repair */
 332                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 333                         rprintf(FINFO,"%s/\n",fname);
 334                 return;
 335         }
 336
 337         if (preserve_links && S_ISLNK(file->mode)) {
 338 #if SUPPORT_LINKS
 339                 char lnk[MAXPATHLEN];
 340                 int l;
 341                 extern int safe_symlinks;
 342
 343                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 344                         if (verbose) {
 345                                 rprintf(FINFO, "ignoring unsafe symlink %s -> \"%s\"\n",
 346                                         full_fname(fname), file->link);
 347                         }
 348                         return;
 349                 }
 350                 if (statret == 0) {
 351                         l = readlink(fname,lnk,MAXPATHLEN-1);
 352                         if (l > 0) {
 353                                 lnk[l] = 0;
 354                                 /* A link already pointing to the
 355                                  * right place -- no further action
 356                                  * required. */
 357                                 if (strcmp(lnk,file->link) == 0) {
 358                                         set_perms(fname,file,&st,1);
 359                                         return;
 360                                 }
 361                         }
 362                         /* Not a symlink, so delete whatever's
 363                          * already there and put a new symlink
 364                          * in place. */
 365                         delete_file(fname);
 366                 }
 367                 if (do_symlink(file->link,fname) != 0) {
 368                         rprintf(FERROR, "symlink %s -> \"%s\" failed: %s\n",
 369                                 full_fname(fname), file->link, strerror(errno));
 370                 } else {
 371                         set_perms(fname,file,NULL,0);
 372                         if (verbose) {
 373                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 374                         }
 375                 }
 376 #endif
 377                 return;
 378         }
 379
 380 #ifdef HAVE_MKNOD
 381         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 382                 if (statret != 0 ||
 383                     st.st_mode != file->mode ||
 384                     (DEV64_T)st.st_rdev != file->rdev) {
 385                         delete_file(fname);
 386                         if (verbose > 2)
 387                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 388                                         fname,(int)file->mode,(int)file->rdev);
 389                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 390                                 rprintf(FERROR, "mknod %s failed: %s\n",
 391                                         full_fname(fname), strerror(errno));
 392                         } else {
 393                                 set_perms(fname,file,NULL,0);
 394                                 if (verbose)
 395                                         rprintf(FINFO,"%s\n",fname);
 396                         }
 397                 } else {
 398                         set_perms(fname,file,&st,1);
 399                 }
 400                 return;
 401         }
 402 #endif
 403
 404         if (preserve_hard_links && check_hard_link(file)) {
 405                 if (verbose > 1) {
 406                         rprintf(FINFO, "recv_generator: \"%s\" is a hard link\n",
 407                                 f_name(file));
 408                 }
 409                 return;
 410         }
 411
 412         if (!S_ISREG(file->mode)) {
 413                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 414                 return;
 415         }
 416
 417         fnamecmp = fname;
 418
 419         if ((statret == -1) && (compare_dest != NULL)) {
 420                 /* try the file at compare_dest instead */
 421                 int saveerrno = errno;
 422                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 423                 statret = link_stat(fnamecmpbuf,&st);
 424                 if (!S_ISREG(st.st_mode))
 425                         statret = -1;
 426                 if (statret == -1)
 427                         errno = saveerrno;
 428 #if HAVE_LINK
 429                 else if (link_dest && !dry_run) {
 430                         if (do_link(fnamecmpbuf, fname) != 0) {
 431                                 if (verbose > 0) {
 432                                         rprintf(FINFO,"link %s => %s : %s\n",
 433                                                 fnamecmpbuf, fname,
 434                                                 strerror(errno));
 435                                 }
 436                         }
 437                         fnamecmp = fnamecmpbuf;
 438                 }
 439 #endif
 440                 else
 441                         fnamecmp = fnamecmpbuf;
 442         }
 443
 444         if (statret == -1) {
 445                 if (errno == ENOENT) {
 446                         write_int(f_out,i);
 447                         if (!dry_run) write_sum_head(f_out, NULL);
 448                 } else if (verbose > 1) {
 449                         rprintf(FERROR,
 450                                 "recv_generator: failed to open %s: %s\n",
 451                                 full_fname(fname), strerror(errno));
 452                 }
 453                 return;
 454         }
 455
 456         if (!S_ISREG(st.st_mode)) {
 457                 if (delete_file(fname) != 0) {
 458                         return;
 459                 }
 460
 461                 /* now pretend the file didn't exist */
 462                 write_int(f_out,i);
 463                 if (!dry_run) write_sum_head(f_out, NULL);
 464                 return;
 465         }
 466
 467         if (opt_ignore_existing && fnamecmp == fname) {
 468                 if (verbose > 1)
 469                         rprintf(FINFO,"%s exists\n",fname);
 470                 return;
 471         }
 472
 473         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 474                 if (verbose > 1)
 475                         rprintf(FINFO,"%s is newer\n",fname);
 476                 return;
 477         }
 478
 479         if (skip_file(fname, file, &st)) {
 480                 if (fnamecmp == fname)
 481                         set_perms(fname,file,&st,1);
 482                 return;
 483         }
 484
 485         if (dry_run) {
 486                 write_int(f_out,i);
 487                 return;
 488         }
 489
 490         if (disable_deltas_p()) {
 491                 write_int(f_out,i);
 492                 write_sum_head(f_out, NULL);
 493                 return;
 494         }
 495
 496         /* open the file */
 497         fd = do_open(fnamecmp, O_RDONLY, 0);
 498
 499         if (fd == -1) {
 500                 rprintf(FERROR, "failed to open %s, continuing: %s\n",
 501                         full_fname(fnamecmp), strerror(errno));
 502                 /* pretend the file didn't exist */
 503                 write_int(f_out,i);
 504                 write_sum_head(f_out, NULL);
 505                 return;
 506         }
 507
 508         if (st.st_size > 0)
 509                 mapbuf = map_file(fd,st.st_size);
 510         else
 511                 mapbuf = NULL;
 512
 513         if (verbose > 3)
 514                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 515
 516         if (verbose > 2)
 517                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 518
 519         write_int(f_out,i);
 520         generate_and_send_sums(mapbuf, st.st_size, f_out);
 521
 522         close(fd);
 523         if (mapbuf) unmap_file(mapbuf);
 524 }
 525
 526
 527
 528 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 529 {
 530         int i;
 531         int phase=0;
 532         char fbuf[MAXPATHLEN];
 533
 534         if (verbose > 2)
 535                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 536                         (int)getpid(),flist->count);
 537
 538         if (verbose >= 2) {
 539                 rprintf(FINFO,
 540                         disable_deltas_p()
 541                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 542                         : "delta transmission enabled\n");
 543         }
 544
 545         /* we expect to just sit around now, so don't exit on a
 546            timeout. If we really get a timeout then the other process should
 547            exit */
 548         io_timeout = 0;
 549
 550         for (i = 0; i < flist->count; i++) {
 551                 struct file_struct *file = flist->files[i];
 552                 mode_t saved_mode = file->mode;
 553                 if (!file->basename) continue;
 554
 555                 /* we need to ensure that any directories we create have writeable
 556                    permissions initially so that we can create the files within
 557                    them. This is then fixed after the files are transferred */
 558                 if (!am_root && S_ISDIR(file->mode)) {
 559                         file->mode |= S_IWUSR; /* user write */
 560                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 561                          * handling of permissions is strange? */
 562                 }
 563
 564                 recv_generator(local_name? local_name
 565                              : f_name_to(file,fbuf,sizeof fbuf), flist, i, f);
 566
 567                 file->mode = saved_mode;
 568         }
 569
 570         phase++;
 571         csum_length = SUM_LENGTH;
 572         ignore_times=1;
 573
 574         if (verbose > 2)
 575                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 576
 577         write_int(f,-1);
 578
 579         /* files can cycle through the system more than once
 580          * to catch initial checksum errors */
 581         for (i = read_int(f_recv); i != -1; i = read_int(f_recv)) {
 582                 struct file_struct *file = flist->files[i];
 583                 recv_generator(local_name? local_name
 584                              : f_name_to(file,fbuf,sizeof fbuf), flist, i, f);
 585         }
 586
 587         phase++;
 588         if (verbose > 2)
 589                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 590
 591         write_int(f,-1);
 592 }