generator.c

   1 /* -*- c-file-style: "linux" -*-
   2
   3    Copyright (C) 1996-2000 by Andrew Tridgell
   4    Copyright (C) Paul Mackerras 1996
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21 #include "rsync.h"
  22
  23 extern int verbose;
  24 extern int dry_run;
  25 extern int relative_paths;
  26 extern int preserve_links;
  27 extern int am_root;
  28 extern int preserve_devices;
  29 extern int preserve_hard_links;
  30 extern int update_only;
  31 extern int whole_file;
  32 extern int block_size;
  33 extern int csum_length;
  34 extern int ignore_times;
  35 extern int size_only;
  36 extern int io_timeout;
  37 extern int remote_version;
  38 extern int always_checksum;
  39 extern int modify_window;
  40 extern char *compare_dest;
  41
  42
  43 /* choose whether to skip a particular file */
  44 static int skip_file(char *fname,
  45                      struct file_struct *file, STRUCT_STAT *st)
  46 {
  47         if (st->st_size != file->length) {
  48                 return 0;
  49         }
  50
  51         /* if always checksum is set then we use the checksum instead
  52            of the file time to determine whether to sync */
  53         if (always_checksum && S_ISREG(st->st_mode)) {
  54                 char sum[MD4_SUM_LENGTH];
  55                 char fnamecmpdest[MAXPATHLEN];
  56
  57                 if (compare_dest != NULL) {
  58                         if (access(fname, 0) != 0) {
  59                                 slprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  60                                                     compare_dest,fname);
  61                                 fname = fnamecmpdest;
  62                         }
  63                 }
  64                 file_checksum(fname,sum,st->st_size);
  65                 if (remote_version < 21) {
  66                         return (memcmp(sum,file->sum,2) == 0);
  67                 } else {
  68                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  69                 }
  70         }
  71
  72         if (size_only) {
  73                 return 1;
  74         }
  75
  76         if (ignore_times) {
  77                 return 0;
  78         }
  79
  80         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  81 }
  82
  83
  84 /* use a larger block size for really big files */
  85 static int adapt_block_size(struct file_struct *file, int bsize)
  86 {
  87         int ret;
  88
  89         if (bsize != BLOCK_SIZE) return bsize;
  90
  91         ret = file->length / (10000); /* rough heuristic */
  92         ret = ret & ~15; /* multiple of 16 */
  93         if (ret < bsize) ret = bsize;
  94         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  95         return ret;
  96 }
  97
  98
  99 /*
 100   send a sums struct down a fd
 101   */
 102 static void send_sums(struct sum_struct *s,int f_out)
 103 {
 104         int i;
 105
 106         /* tell the other guy how many we are going to be doing and how many
 107            bytes there are in the last chunk */
 108         write_int(f_out,s?s->count:0);
 109         write_int(f_out,s?s->n:block_size);
 110         write_int(f_out,s?s->remainder:0);
 111
 112         if (!s) return;
 113
 114         for (i=0;i<s->count;i++) {
 115                 write_int(f_out,s->sums[i].sum1);
 116                 write_buf(f_out,s->sums[i].sum2,csum_length);
 117         }
 118 }
 119
 120
 121 /*
 122   generate a stream of signatures/checksums that describe a buffer
 123
 124   generate approximately one checksum every n bytes
 125   */
 126 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 127 {
 128         int i;
 129         struct sum_struct *s;
 130         int count;
 131         int block_len = n;
 132         int remainder = (len%block_len);
 133         OFF_T offset = 0;
 134
 135         count = (len+(block_len-1))/block_len;
 136
 137         s = (struct sum_struct *)malloc(sizeof(*s));
 138         if (!s) out_of_memory("generate_sums");
 139
 140         s->count = count;
 141         s->remainder = remainder;
 142         s->n = n;
 143         s->flength = len;
 144
 145         if (count==0) {
 146                 s->sums = NULL;
 147                 return s;
 148         }
 149
 150         if (verbose > 3)
 151                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 152                         s->count,s->remainder,s->n,(double)s->flength);
 153
 154         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 155         if (!s->sums) out_of_memory("generate_sums");
 156
 157         for (i=0;i<count;i++) {
 158                 int n1 = MIN(len,n);
 159                 char *map = map_ptr(buf,offset,n1);
 160
 161                 s->sums[i].sum1 = get_checksum1(map,n1);
 162                 get_checksum2(map,n1,s->sums[i].sum2);
 163
 164                 s->sums[i].offset = offset;
 165                 s->sums[i].len = n1;
 166                 s->sums[i].i = i;
 167
 168                 if (verbose > 3)
 169                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 170                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 171
 172                 len -= n1;
 173                 offset += n1;
 174         }
 175
 176         return s;
 177 }
 178
 179
 180
 181 /*
 182  * Acts on file number I from FLIST, whose name is fname.
 183  *
 184  * First fixes up permissions, then generates checksums for the file.
 185  *
 186  * (This comment was added later by mbp who was trying to work it out;
 187  * it might be wrong.)
 188  */
 189 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 190 {
 191         int fd;
 192         STRUCT_STAT st;
 193         struct map_struct *buf;
 194         struct sum_struct *s;
 195         int statret;
 196         struct file_struct *file = flist->files[i];
 197         char *fnamecmp;
 198         char fnamecmpbuf[MAXPATHLEN];
 199         extern char *compare_dest;
 200         extern int list_only;
 201         extern int preserve_perms;
 202         extern int only_existing;
 203
 204         if (list_only) return;
 205
 206         if (verbose > 2)
 207                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 208
 209         statret = link_stat(fname,&st);
 210
 211         if (only_existing && statret == -1 && errno == ENOENT) {
 212                 /* we only want to update existing files */
 213                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 214                 return;
 215         }
 216
 217         if (statret == 0 &&
 218             !preserve_perms &&
 219             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 220                 /* if the file exists already and we aren't perserving
 221                    presmissions then act as though the remote end sent
 222                    us the file permissions we already have */
 223                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 224         }
 225
 226         if (S_ISDIR(file->mode)) {
 227                 if (dry_run) return;
 228                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 229                         if (robust_unlink(fname) != 0) {
 230                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",fname,strerror(errno));
 231                                 return;
 232                         }
 233                         statret = -1;
 234                 }
 235                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 236                         if (!(relative_paths && errno==ENOENT &&
 237                               create_directory_path(fname)==0 &&
 238                               do_mkdir(fname,file->mode)==0)) {
 239                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
 240                                         fname,strerror(errno));
 241                         }
 242                 }
 243                 if (set_perms(fname,file,NULL,0) && verbose)
 244                         rprintf(FINFO,"%s/\n",fname);
 245                 return;
 246         }
 247
 248         if (preserve_links && S_ISLNK(file->mode)) {
 249 #if SUPPORT_LINKS
 250                 char lnk[MAXPATHLEN];
 251                 int l;
 252                 extern int safe_symlinks;
 253
 254                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 255                         if (verbose) {
 256                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 257                                         fname,file->link);
 258                         }
 259                         return;
 260                 }
 261                 if (statret == 0) {
 262                         l = readlink(fname,lnk,MAXPATHLEN-1);
 263                         if (l > 0) {
 264                                 lnk[l] = 0;
 265                                 if (strcmp(lnk,file->link) == 0) {
 266                                         set_perms(fname,file,&st,1);
 267                                         return;
 268                                 }
 269                         }
 270                         delete_file(fname);
 271                 }
 272                 if (do_symlink(file->link,fname) != 0) {
 273                         rprintf(FERROR,"symlink %s -> %s : %s\n",
 274                                 fname,file->link,strerror(errno));
 275                 } else {
 276                         set_perms(fname,file,NULL,0);
 277                         if (verbose) {
 278                                 rprintf(FINFO,"%s -> %s\n",
 279                                         fname,file->link);
 280                         }
 281                 }
 282 #endif
 283                 return;
 284         }
 285
 286 #ifdef HAVE_MKNOD
 287         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 288                 if (statret != 0 ||
 289                     st.st_mode != file->mode ||
 290                     st.st_rdev != file->rdev) {
 291                         delete_file(fname);
 292                         if (verbose > 2)
 293                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 294                                         fname,(int)file->mode,(int)file->rdev);
 295                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 296                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 297                         } else {
 298                                 set_perms(fname,file,NULL,0);
 299                                 if (verbose)
 300                                         rprintf(FINFO,"%s\n",fname);
 301                         }
 302                 } else {
 303                         set_perms(fname,file,&st,1);
 304                 }
 305                 return;
 306         }
 307 #endif
 308
 309         if (preserve_hard_links && check_hard_link(file)) {
 310                 if (verbose > 1)
 311                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 312                 return;
 313         }
 314
 315         if (!S_ISREG(file->mode)) {
 316                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 317                 return;
 318         }
 319
 320         fnamecmp = fname;
 321
 322         if ((statret == -1) && (compare_dest != NULL)) {
 323                 /* try the file at compare_dest instead */
 324                 int saveerrno = errno;
 325                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 326                 statret = link_stat(fnamecmpbuf,&st);
 327                 if (!S_ISREG(st.st_mode))
 328                         statret = -1;
 329                 if (statret == -1)
 330                         errno = saveerrno;
 331                 else
 332                         fnamecmp = fnamecmpbuf;
 333         }
 334
 335         if (statret == -1) {
 336                 if (errno == ENOENT) {
 337                         write_int(f_out,i);
 338                         if (!dry_run) send_sums(NULL,f_out);
 339                 } else {
 340                         if (verbose > 1)
 341                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 342                 }
 343                 return;
 344         }
 345
 346         if (!S_ISREG(st.st_mode)) {
 347                 if (delete_file(fname) != 0) {
 348                         return;
 349                 }
 350
 351                 /* now pretend the file didn't exist */
 352                 write_int(f_out,i);
 353                 if (!dry_run) send_sums(NULL,f_out);
 354                 return;
 355         }
 356
 357         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 358                 if (verbose > 1)
 359                         rprintf(FINFO,"%s is newer\n",fname);
 360                 return;
 361         }
 362
 363         if (skip_file(fname, file, &st)) {
 364                 if (fnamecmp == fname)
 365                         set_perms(fname,file,&st,1);
 366                 return;
 367         }
 368
 369         if (dry_run) {
 370                 write_int(f_out,i);
 371                 return;
 372         }
 373
 374         if (whole_file) {
 375                 write_int(f_out,i);
 376                 send_sums(NULL,f_out);
 377                 return;
 378         }
 379
 380         /* open the file */
 381         fd = do_open(fnamecmp, O_RDONLY, 0);
 382
 383         if (fd == -1) {
 384                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 385                 /* pretend the file didn't exist */
 386                 write_int(f_out,i);
 387                 send_sums(NULL,f_out);
 388                 return;
 389         }
 390
 391         if (st.st_size > 0) {
 392                 buf = map_file(fd,st.st_size);
 393         } else {
 394                 buf = NULL;
 395         }
 396
 397         if (verbose > 3)
 398                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 399
 400         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 401
 402         if (verbose > 2)
 403                 rprintf(FINFO,"sending sums for %d\n",i);
 404
 405         write_int(f_out,i);
 406         send_sums(s,f_out);
 407
 408         close(fd);
 409         if (buf) unmap_file(buf);
 410
 411         free_sums(s);
 412 }
 413
 414
 415
 416 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 417 {
 418         int i;
 419         int phase=0;
 420
 421         if (verbose > 2)
 422                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 423                         (int)getpid(),flist->count);
 424
 425         for (i = 0; i < flist->count; i++) {
 426                 struct file_struct *file = flist->files[i];
 427                 mode_t saved_mode = file->mode;
 428                 if (!file->basename) continue;
 429
 430                 /* we need to ensure that any directories we create have writeable
 431                    permissions initially so that we can create the files within
 432                    them. This is then fixed after the files are transferred */
 433                 if (!am_root && S_ISDIR(file->mode)) {
 434                         file->mode |= S_IWUSR; /* user write */
 435                 }
 436
 437                 recv_generator(local_name?local_name:f_name(file),
 438                                flist,i,f);
 439
 440                 file->mode = saved_mode;
 441         }
 442
 443         phase++;
 444         csum_length = SUM_LENGTH;
 445         ignore_times=1;
 446
 447         if (verbose > 2)
 448                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 449
 450         write_int(f,-1);
 451
 452         /* we expect to just sit around now, so don't exit on a
 453            timeout. If we really get a timeout then the other process should
 454            exit */
 455         io_timeout = 0;
 456
 457         if (remote_version >= 13) {
 458                 /* in newer versions of the protocol the files can cycle through
 459                    the system more than once to catch initial checksum errors */
 460                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 461                         struct file_struct *file = flist->files[i];
 462                         recv_generator(local_name?local_name:f_name(file),
 463                                        flist,i,f);
 464                 }
 465
 466                 phase++;
 467                 if (verbose > 2)
 468                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 469
 470                 write_int(f,-1);
 471         }
 472 }