generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    rsync -- fast file replication program
   4
   5    Copyright (C) 1996-2000 by Andrew Tridgell
   6    Copyright (C) Paul Mackerras 1996
   7    Copyright (C) 2002 by Martin Pool <mbp@samba.org>
   8
   9    This program is free software; you can redistribute it and/or modify
  10    it under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 2 of the License, or
  12    (at your option) any later version.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program; if not, write to the Free Software
  21    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22 */
  23
  24 #include "rsync.h"
  25
  26 extern int verbose;
  27 extern int dry_run;
  28 extern int relative_paths;
  29 extern int preserve_links;
  30 extern int am_root;
  31 extern int preserve_devices;
  32 extern int preserve_hard_links;
  33 extern int update_only;
  34 extern int opt_ignore_existing;
  35 extern int block_size;
  36 extern int csum_length;
  37 extern int ignore_times;
  38 extern int size_only;
  39 extern int io_timeout;
  40 extern int remote_version;
  41 extern int always_checksum;
  42 extern int modify_window;
  43 extern char *compare_dest;
  44 extern int link_dest;
  45
  46
  47 /* choose whether to skip a particular file */
  48 static int skip_file(char *fname,
  49                      struct file_struct *file, STRUCT_STAT *st)
  50 {
  51         if (st->st_size != file->length) {
  52                 return 0;
  53         }
  54         if (link_dest) {
  55                 extern int preserve_perms;
  56                 extern int preserve_uid;
  57                 extern int preserve_gid;
  58
  59                 if(preserve_perms
  60                     && (st->st_mode & ~_S_IFMT) !=  (file->mode & ~_S_IFMT))
  61                         return 0;
  62
  63                 if (preserve_uid && st->st_uid != file->uid)
  64                         return 0;
  65
  66                 if (preserve_gid && st->st_gid != file->gid)
  67                         return 0;
  68         }
  69
  70         /* if always checksum is set then we use the checksum instead
  71            of the file time to determine whether to sync */
  72         if (always_checksum && S_ISREG(st->st_mode)) {
  73                 char sum[MD4_SUM_LENGTH];
  74                 char fnamecmpdest[MAXPATHLEN];
  75
  76                 if (compare_dest != NULL) {
  77                         if (access(fname, 0) != 0) {
  78                                 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  79                                          compare_dest,fname);
  80                                 fname = fnamecmpdest;
  81                         }
  82                 }
  83                 file_checksum(fname,sum,st->st_size);
  84                 if (remote_version < 21) {
  85                         return (memcmp(sum,file->sum,2) == 0);
  86                 } else {
  87                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  88                 }
  89         }
  90
  91         if (size_only) {
  92                 return 1;
  93         }
  94
  95         if (ignore_times) {
  96                 return 0;
  97         }
  98
  99         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
 100 }
 101
 102
 103 /* use a larger block size for really big files */
 104 static int adapt_block_size(struct file_struct *file, int bsize)
 105 {
 106         int ret;
 107
 108         if (bsize != BLOCK_SIZE) return bsize;
 109
 110         ret = file->length / (10000); /* rough heuristic */
 111         ret = ret & ~15; /* multiple of 16 */
 112         if (ret < bsize) ret = bsize;
 113         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
 114         return ret;
 115 }
 116
 117
 118 /*
 119  *      NULL sum_struct means we have no checksums
 120   */
 121
 122 void write_sum_head(int f, struct sum_struct *sum)
 123 {
 124         static struct sum_struct null_sum;
 125
 126         if (sum == (struct sum_struct *)NULL)
 127                 sum = &null_sum;
 128
 129         write_int(f, sum->count);
 130         write_int(f, sum->blength);
 131         if (remote_version >= 27)
 132                 write_int(f, sum->s2length);
 133         write_int(f, sum->remainder);
 134 }
 135
 136
 137
 138 /**
 139  * Perhaps we want to just send an empty checksum set for this file,
 140  * which will force the whole thing to be literally transferred.
 141  *
 142  * When do we do this?  If the user's explicitly said they
 143  * want the whole thing, or if { they haven't explicitly
 144  * requested a delta, and it's local but not batch mode.}
 145  *
 146  * Whew. */
 147 static BOOL disable_deltas_p(void)
 148 {
 149         extern int whole_file;
 150         extern int local_server;
 151         extern int write_batch;
 152
 153         if (whole_file > 0)
 154                 return True;
 155         if (whole_file == 0 || write_batch)
 156                 return False;
 157         return local_server;
 158 }
 159
 160
 161 /*
 162  * Generate and send a stream of signatures/checksums that describe a buffer
 163  *
 164  * Generate approximately one checksum every block_len bytes.
 165  */
 166 static void generate_and_send_sums(struct map_struct *buf, OFF_T len,
 167                                    int block_len, int f_out)
 168 {
 169         size_t i;
 170         struct sum_struct sum;
 171         OFF_T offset = 0;
 172
 173         sum.count = (len + (block_len - 1)) / block_len;
 174         sum.remainder = (len % block_len);
 175         sum.blength     = block_len;
 176         sum.flength = len;
 177         sum.s2length    = csum_length;
 178         /* not needed here  sum.sums = NULL; */
 179
 180         if (sum.count && verbose > 3) {
 181                 rprintf(FINFO, "count=%ld rem=%ld n=%ld flength=%.0f\n",
 182                         (long) sum.count, (long) sum.remainder,
 183                         (long) sum.blength, (double) sum.flength);
 184         }
 185
 186         write_sum_head(f_out, &sum);
 187
 188         for (i = 0; i < sum.count; i++) {
 189                 int n1 = MIN(len, block_len);
 190                 char *map = map_ptr(buf, offset, n1);
 191                 uint32 sum1 = get_checksum1(map, n1);
 192                 char sum2[SUM_LENGTH];
 193
 194                 get_checksum2(map, n1, sum2);
 195
 196                 if (verbose > 3) {
 197                         rprintf(FINFO,
 198                                 "chunk[%d] offset=%.0f len=%d sum1=%08lx\n",
 199                                 i, (double) offset, n1, (unsigned long) sum1);
 200                 }
 201                 write_int(f_out, sum1);
 202                 write_buf(f_out, sum2, sum.s2length);
 203                 len -= n1;
 204                 offset += n1;
 205         }
 206 }
 207
 208
 209
 210 /**
 211  * Acts on file number @p i from @p flist, whose name is @p fname.
 212  *
 213  * First fixes up permissions, then generates checksums for the file.
 214  *
 215  * @note This comment was added later by mbp who was trying to work it
 216  * out.  It might be wrong.
 217  **/
 218 void recv_generator(char *fname, struct file_list *flist, int i, int f_out)
 219 {
 220         int fd;
 221         STRUCT_STAT st;
 222         struct map_struct *buf;
 223         int statret;
 224         struct file_struct *file = flist->files[i];
 225         char *fnamecmp;
 226         char fnamecmpbuf[MAXPATHLEN];
 227         extern char *compare_dest;
 228         extern int list_only;
 229         extern int preserve_perms;
 230         extern int only_existing;
 231         extern int orig_umask;
 232
 233         if (list_only) return;
 234
 235         if (verbose > 2)
 236                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 237
 238         statret = link_stat(fname,&st);
 239
 240         if (only_existing && statret == -1 && errno == ENOENT) {
 241                 /* we only want to update existing files */
 242                 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
 243                 return;
 244         }
 245
 246         if (statret == 0 &&
 247             !preserve_perms &&
 248             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 249                 /* if the file exists already and we aren't perserving
 250                  * permissions then act as though the remote end sent
 251                  * us the file permissions we already have */
 252                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 253         }
 254
 255         if (S_ISDIR(file->mode)) {
 256                 /* The file to be received is a directory, so we need
 257                  * to prepare appropriately.  If there is already a
 258                  * file of that name and it is *not* a directory, then
 259                  * we need to delete it.  If it doesn't exist, then
 260                  * recursively create it. */
 261
 262                 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
 263                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 264                         if (robust_unlink(fname) != 0) {
 265                                 rprintf(FERROR, RSYNC_NAME
 266                                         ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
 267                                         fname,strerror(errno));
 268                                 return;
 269                         }
 270                         statret = -1;
 271                 }
 272                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 273                         if (!(relative_paths && errno==ENOENT &&
 274                               create_directory_path(fname, orig_umask)==0 &&
 275                               do_mkdir(fname,file->mode)==0)) {
 276                                 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
 277                                         fname,strerror(errno));
 278                         }
 279                 }
 280                 /* f_out is set to -1 when doing final directory
 281                    permission and modification time repair */
 282                 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
 283                         rprintf(FINFO,"%s/\n",fname);
 284                 return;
 285         }
 286
 287         if (preserve_links && S_ISLNK(file->mode)) {
 288 #if SUPPORT_LINKS
 289                 char lnk[MAXPATHLEN];
 290                 int l;
 291                 extern int safe_symlinks;
 292
 293                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 294                         if (verbose) {
 295                                 rprintf(FINFO,"ignoring unsafe symlink \"%s\" -> \"%s\"\n",
 296                                         fname,file->link);
 297                         }
 298                         return;
 299                 }
 300                 if (statret == 0) {
 301                         l = readlink(fname,lnk,MAXPATHLEN-1);
 302                         if (l > 0) {
 303                                 lnk[l] = 0;
 304                                 /* A link already pointing to the
 305                                  * right place -- no further action
 306                                  * required. */
 307                                 if (strcmp(lnk,file->link) == 0) {
 308                                         set_perms(fname,file,&st,1);
 309                                         return;
 310                                 }
 311                         }
 312                         /* Not a symlink, so delete whatever's
 313                          * already there and put a new symlink
 314                          * in place. */
 315                         delete_file(fname);
 316                 }
 317                 if (do_symlink(file->link,fname) != 0) {
 318                         rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
 319                                 fname,file->link,strerror(errno));
 320                 } else {
 321                         set_perms(fname,file,NULL,0);
 322                         if (verbose) {
 323                                 rprintf(FINFO,"%s -> %s\n", fname,file->link);
 324                         }
 325                 }
 326 #endif
 327                 return;
 328         }
 329
 330 #ifdef HAVE_MKNOD
 331         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 332                 if (statret != 0 ||
 333                     st.st_mode != file->mode ||
 334                     st.st_rdev != file->rdev) {
 335                         delete_file(fname);
 336                         if (verbose > 2)
 337                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 338                                         fname,(int)file->mode,(int)file->rdev);
 339                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 340                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 341                         } else {
 342                                 set_perms(fname,file,NULL,0);
 343                                 if (verbose)
 344                                         rprintf(FINFO,"%s\n",fname);
 345                         }
 346                 } else {
 347                         set_perms(fname,file,&st,1);
 348                 }
 349                 return;
 350         }
 351 #endif
 352
 353         if (preserve_hard_links && check_hard_link(file)) {
 354                 if (verbose > 1)
 355                         rprintf(FINFO, "recv_generator: \"%s\" is a hard link\n",f_name(file));
 356                 return;
 357         }
 358
 359         if (!S_ISREG(file->mode)) {
 360                 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
 361                 return;
 362         }
 363
 364         fnamecmp = fname;
 365
 366         if ((statret == -1) && (compare_dest != NULL)) {
 367                 /* try the file at compare_dest instead */
 368                 int saveerrno = errno;
 369                 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 370                 statret = link_stat(fnamecmpbuf,&st);
 371                 if (!S_ISREG(st.st_mode))
 372                         statret = -1;
 373                 if (statret == -1)
 374                         errno = saveerrno;
 375 #if HAVE_LINK
 376                 else if (link_dest && !dry_run) {
 377                         if (do_link(fnamecmpbuf, fname) != 0) {
 378                                 if (verbose > 0)
 379                                         rprintf(FINFO,"link %s => %s : %s\n",
 380                                                 fnamecmpbuf,
 381                                                 fname,
 382                                                 strerror(errno));
 383                         }
 384                         fnamecmp = fnamecmpbuf;
 385                 }
 386 #endif
 387                 else
 388                         fnamecmp = fnamecmpbuf;
 389         }
 390
 391         if (statret == -1) {
 392                 if (errno == ENOENT) {
 393                         write_int(f_out,i);
 394                         if (!dry_run) write_sum_head(f_out, NULL);
 395                 } else {
 396                         if (verbose > 1)
 397                                 rprintf(FERROR, RSYNC_NAME
 398                                         ": recv_generator failed to open \"%s\": %s\n",
 399                                         fname, strerror(errno));
 400                 }
 401                 return;
 402         }
 403
 404         if (!S_ISREG(st.st_mode)) {
 405                 if (delete_file(fname) != 0) {
 406                         return;
 407                 }
 408
 409                 /* now pretend the file didn't exist */
 410                 write_int(f_out,i);
 411                 if (!dry_run) write_sum_head(f_out, NULL);
 412                 return;
 413         }
 414
 415         if (opt_ignore_existing && fnamecmp == fname) {
 416                 if (verbose > 1)
 417                         rprintf(FINFO,"%s exists\n",fname);
 418                 return;
 419         }
 420
 421         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 422                 if (verbose > 1)
 423                         rprintf(FINFO,"%s is newer\n",fname);
 424                 return;
 425         }
 426
 427         if (skip_file(fname, file, &st)) {
 428                 if (fnamecmp == fname)
 429                         set_perms(fname,file,&st,1);
 430                 return;
 431         }
 432
 433         if (dry_run) {
 434                 write_int(f_out,i);
 435                 return;
 436         }
 437
 438         if (disable_deltas_p()) {
 439                 write_int(f_out,i);
 440                 write_sum_head(f_out, NULL);
 441                 return;
 442         }
 443
 444         /* open the file */
 445         fd = do_open(fnamecmp, O_RDONLY, 0);
 446
 447         if (fd == -1) {
 448                 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
 449                 /* pretend the file didn't exist */
 450                 write_int(f_out,i);
 451                 write_sum_head(f_out, NULL);
 452                 return;
 453         }
 454
 455         if (st.st_size > 0) {
 456                 buf = map_file(fd,st.st_size);
 457         } else {
 458                 buf = NULL;
 459         }
 460
 461         if (verbose > 3)
 462                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 463
 464         if (verbose > 2)
 465                 rprintf(FINFO, "generating and sending sums for %d\n", i);
 466
 467         write_int(f_out,i);
 468         generate_and_send_sums(buf, st.st_size,
 469                                adapt_block_size(file, block_size), f_out);
 470
 471         close(fd);
 472         if (buf) unmap_file(buf);
 473 }
 474
 475
 476
 477 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 478 {
 479         int i;
 480         int phase=0;
 481
 482         if (verbose > 2)
 483                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 484                         (int)getpid(),flist->count);
 485
 486         if (verbose >= 2) {
 487                 rprintf(FINFO,
 488                         disable_deltas_p()
 489                         ? "delta-transmission disabled for local transfer or --whole-file\n"
 490                         : "delta transmission enabled\n");
 491         }
 492
 493         /* we expect to just sit around now, so don't exit on a
 494            timeout. If we really get a timeout then the other process should
 495            exit */
 496         io_timeout = 0;
 497
 498         for (i = 0; i < flist->count; i++) {
 499                 struct file_struct *file = flist->files[i];
 500                 mode_t saved_mode = file->mode;
 501                 if (!file->basename) continue;
 502
 503                 /* we need to ensure that any directories we create have writeable
 504                    permissions initially so that we can create the files within
 505                    them. This is then fixed after the files are transferred */
 506                 if (!am_root && S_ISDIR(file->mode)) {
 507                         file->mode |= S_IWUSR; /* user write */
 508                         /* XXX: Could this be causing a problem on SCO?  Perhaps their
 509                          * handling of permissions is strange? */
 510                 }
 511
 512                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 513
 514                 file->mode = saved_mode;
 515         }
 516
 517         phase++;
 518         csum_length = SUM_LENGTH;
 519         ignore_times=1;
 520
 521         if (verbose > 2)
 522                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 523
 524         write_int(f,-1);
 525
 526         /* files can cycle through the system more than once
 527          * to catch initial checksum errors */
 528         for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 529                 struct file_struct *file = flist->files[i];
 530                 recv_generator(local_name?local_name:f_name(file), flist,i,f);
 531         }
 532
 533         phase++;
 534         if (verbose > 2)
 535                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 536
 537         write_int(f,-1);
 538 }