generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38 extern int modify_window;
  39 extern char *compare_dest;
  40
  41
  42 /* choose whether to skip a particular file */
  43 static int skip_file(char *fname,
  44                      struct file_struct *file, STRUCT_STAT *st)
  45 {
  46         if (st->st_size != file->length) {
  47                 return 0;
  48         }
  49
  50         /* if always checksum is set then we use the checksum instead
  51            of the file time to determine whether to sync */
  52         if (always_checksum && S_ISREG(st->st_mode)) {
  53                 char sum[MD4_SUM_LENGTH];
  54                 char fnamecmpdest[MAXPATHLEN];
  55
  56                 if (compare_dest != NULL) {
  57                         if (access(fname, 0) != 0) {
  58                                 slprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  59                                                     compare_dest,fname);
  60                                 fname = fnamecmpdest;
  61                         }
  62                 }
  63                 file_checksum(fname,sum,st->st_size);
  64                 if (remote_version < 21) {
  65                         return (memcmp(sum,file->sum,2) == 0);
  66                 } else {
  67                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  68                 }
  69         }
  70
  71         if (size_only) {
  72                 return 1;
  73         }
  74
  75         if (ignore_times) {
  76                 return 0;
  77         }
  78
  79         return (cmp_modtime(st->st_mtime,file->modtime) == 0);
  80 }
  81
  82
  83 /* use a larger block size for really big files */
  84 static int adapt_block_size(struct file_struct *file, int bsize)
  85 {
  86         int ret;
  87
  88         if (bsize != BLOCK_SIZE) return bsize;
  89
  90         ret = file->length / (10000); /* rough heuristic */
  91         ret = ret & ~15; /* multiple of 16 */
  92         if (ret < bsize) ret = bsize;
  93         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  94         return ret;
  95 }
  96
  97
  98 /*
  99   send a sums struct down a fd
 100   */
 101 static void send_sums(struct sum_struct *s,int f_out)
 102 {
 103         int i;
 104
 105         /* tell the other guy how many we are going to be doing and how many
 106            bytes there are in the last chunk */
 107         write_int(f_out,s?s->count:0);
 108         write_int(f_out,s?s->n:block_size);
 109         write_int(f_out,s?s->remainder:0);
 110
 111         if (!s) return;
 112
 113         for (i=0;i<s->count;i++) {
 114                 write_int(f_out,s->sums[i].sum1);
 115                 write_buf(f_out,s->sums[i].sum2,csum_length);
 116         }
 117 }
 118
 119
 120 /*
 121   generate a stream of signatures/checksums that describe a buffer
 122
 123   generate approximately one checksum every n bytes
 124   */
 125 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 126 {
 127         int i;
 128         struct sum_struct *s;
 129         int count;
 130         int block_len = n;
 131         int remainder = (len%block_len);
 132         OFF_T offset = 0;
 133
 134         count = (len+(block_len-1))/block_len;
 135
 136         s = (struct sum_struct *)malloc(sizeof(*s));
 137         if (!s) out_of_memory("generate_sums");
 138
 139         s->count = count;
 140         s->remainder = remainder;
 141         s->n = n;
 142         s->flength = len;
 143
 144         if (count==0) {
 145                 s->sums = NULL;
 146                 return s;
 147         }
 148
 149         if (verbose > 3)
 150                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 151                         s->count,s->remainder,s->n,(double)s->flength);
 152
 153         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 154         if (!s->sums) out_of_memory("generate_sums");
 155
 156         for (i=0;i<count;i++) {
 157                 int n1 = MIN(len,n);
 158                 char *map = map_ptr(buf,offset,n1);
 159
 160                 s->sums[i].sum1 = get_checksum1(map,n1);
 161                 get_checksum2(map,n1,s->sums[i].sum2);
 162
 163                 s->sums[i].offset = offset;
 164                 s->sums[i].len = n1;
 165                 s->sums[i].i = i;
 166
 167                 if (verbose > 3)
 168                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 169                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 170
 171                 len -= n1;
 172                 offset += n1;
 173         }
 174
 175         return s;
 176 }
 177
 178
 179 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 180 {
 181         int fd;
 182         STRUCT_STAT st;
 183         struct map_struct *buf;
 184         struct sum_struct *s;
 185         int statret;
 186         struct file_struct *file = flist->files[i];
 187         char *fnamecmp;
 188         char fnamecmpbuf[MAXPATHLEN];
 189         extern char *compare_dest;
 190         extern int list_only;
 191         extern int preserve_perms;
 192         extern int only_existing;
 193
 194         if (list_only) return;
 195
 196         if (verbose > 2)
 197                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 198
 199         statret = link_stat(fname,&st);
 200
 201         if (only_existing && statret == -1 && errno == ENOENT) {
 202                 /* we only want to update existing files */
 203                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 204                 return;
 205         }
 206
 207         if (statret == 0 &&
 208             !preserve_perms &&
 209             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 210                 /* if the file exists already and we aren't perserving
 211                    presmissions then act as though the remote end sent
 212                    us the file permissions we already have */
 213                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 214         }
 215
 216         if (S_ISDIR(file->mode)) {
 217                 if (dry_run) return;
 218                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 219                         if (robust_unlink(fname) != 0) {
 220                                 rprintf(FERROR,"recv_generator: unlink %s: %s\n",fname,strerror(errno));
 221                                 return;
 222                         }
 223                         statret = -1;
 224                 }
 225                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 226                         if (!(relative_paths && errno==ENOENT &&
 227                               create_directory_path(fname)==0 &&
 228                               do_mkdir(fname,file->mode)==0)) {
 229                                 rprintf(FERROR,"recv_generator: mkdir %s: %s (2)\n",
 230                                         fname,strerror(errno));
 231                         }
 232                 }
 233                 if (set_perms(fname,file,NULL,0) && verbose)
 234                         rprintf(FINFO,"%s/\n",fname);
 235                 return;
 236         }
 237
 238         if (preserve_links && S_ISLNK(file->mode)) {
 239 #if SUPPORT_LINKS
 240                 char lnk[MAXPATHLEN];
 241                 int l;
 242                 extern int safe_symlinks;
 243
 244                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 245                         if (verbose) {
 246                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 247                                         fname,file->link);
 248                         }
 249                         return;
 250                 }
 251                 if (statret == 0) {
 252                         l = readlink(fname,lnk,MAXPATHLEN-1);
 253                         if (l > 0) {
 254                                 lnk[l] = 0;
 255                                 if (strcmp(lnk,file->link) == 0) {
 256                                         set_perms(fname,file,&st,1);
 257                                         return;
 258                                 }
 259                         }
 260                         delete_file(fname);
 261                 }
 262                 if (do_symlink(file->link,fname) != 0) {
 263                         rprintf(FERROR,"symlink %s -> %s : %s\n",
 264                                 fname,file->link,strerror(errno));
 265                 } else {
 266                         set_perms(fname,file,NULL,0);
 267                         if (verbose) {
 268                                 rprintf(FINFO,"%s -> %s\n",
 269                                         fname,file->link);
 270                         }
 271                 }
 272 #endif
 273                 return;
 274         }
 275
 276 #ifdef HAVE_MKNOD
 277         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 278                 if (statret != 0 ||
 279                     st.st_mode != file->mode ||
 280                     st.st_rdev != file->rdev) {
 281                         delete_file(fname);
 282                         if (verbose > 2)
 283                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 284                                         fname,(int)file->mode,(int)file->rdev);
 285                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 286                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 287                         } else {
 288                                 set_perms(fname,file,NULL,0);
 289                                 if (verbose)
 290                                         rprintf(FINFO,"%s\n",fname);
 291                         }
 292                 } else {
 293                         set_perms(fname,file,&st,1);
 294                 }
 295                 return;
 296         }
 297 #endif
 298
 299         if (preserve_hard_links && check_hard_link(file)) {
 300                 if (verbose > 1)
 301                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 302                 return;
 303         }
 304
 305         if (!S_ISREG(file->mode)) {
 306                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 307                 return;
 308         }
 309
 310         fnamecmp = fname;
 311
 312         if ((statret == -1) && (compare_dest != NULL)) {
 313                 /* try the file at compare_dest instead */
 314                 int saveerrno = errno;
 315                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 316                 statret = link_stat(fnamecmpbuf,&st);
 317                 if (!S_ISREG(st.st_mode))
 318                         statret = -1;
 319                 if (statret == -1)
 320                         errno = saveerrno;
 321                 else
 322                         fnamecmp = fnamecmpbuf;
 323         }
 324
 325         if (statret == -1) {
 326                 if (errno == ENOENT) {
 327                         write_int(f_out,i);
 328                         if (!dry_run) send_sums(NULL,f_out);
 329                 } else {
 330                         if (verbose > 1)
 331                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 332                 }
 333                 return;
 334         }
 335
 336         if (!S_ISREG(st.st_mode)) {
 337                 if (delete_file(fname) != 0) {
 338                         return;
 339                 }
 340
 341                 /* now pretend the file didn't exist */
 342                 write_int(f_out,i);
 343                 if (!dry_run) send_sums(NULL,f_out);
 344                 return;
 345         }
 346
 347         if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
 348                 if (verbose > 1)
 349                         rprintf(FINFO,"%s is newer\n",fname);
 350                 return;
 351         }
 352
 353         if (skip_file(fname, file, &st)) {
 354                 if (fnamecmp == fname)
 355                         set_perms(fname,file,&st,1);
 356                 return;
 357         }
 358
 359         if (dry_run) {
 360                 write_int(f_out,i);
 361                 return;
 362         }
 363
 364         if (whole_file) {
 365                 write_int(f_out,i);
 366                 send_sums(NULL,f_out);
 367                 return;
 368         }
 369
 370         /* open the file */
 371         fd = do_open(fnamecmp, O_RDONLY, 0);
 372
 373         if (fd == -1) {
 374                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 375                 /* pretend the file didn't exist */
 376                 write_int(f_out,i);
 377                 send_sums(NULL,f_out);
 378                 return;
 379         }
 380
 381         if (st.st_size > 0) {
 382                 buf = map_file(fd,st.st_size);
 383         } else {
 384                 buf = NULL;
 385         }
 386
 387         if (verbose > 3)
 388                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 389
 390         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 391
 392         if (verbose > 2)
 393                 rprintf(FINFO,"sending sums for %d\n",i);
 394
 395         write_int(f_out,i);
 396         send_sums(s,f_out);
 397
 398         close(fd);
 399         if (buf) unmap_file(buf);
 400
 401         free_sums(s);
 402 }
 403
 404
 405
 406 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 407 {
 408         int i;
 409         int phase=0;
 410
 411         if (verbose > 2)
 412                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 413                         (int)getpid(),flist->count);
 414
 415         for (i = 0; i < flist->count; i++) {
 416                 struct file_struct *file = flist->files[i];
 417                 mode_t saved_mode = file->mode;
 418                 if (!file->basename) continue;
 419
 420                 /* we need to ensure that any directories we create have writeable
 421                    permissions initially so that we can create the files within
 422                    them. This is then fixed after the files are transferred */
 423                 if (!am_root && S_ISDIR(file->mode)) {
 424                         file->mode |= S_IWUSR; /* user write */
 425                 }
 426
 427                 recv_generator(local_name?local_name:f_name(file),
 428                                flist,i,f);
 429
 430                 file->mode = saved_mode;
 431         }
 432
 433         phase++;
 434         csum_length = SUM_LENGTH;
 435         ignore_times=1;
 436
 437         if (verbose > 2)
 438                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 439
 440         write_int(f,-1);
 441
 442         /* we expect to just sit around now, so don't exit on a
 443            timeout. If we really get a timeout then the other process should
 444            exit */
 445         io_timeout = 0;
 446
 447         if (remote_version >= 13) {
 448                 /* in newer versions of the protocol the files can cycle through
 449                    the system more than once to catch initial checksum errors */
 450                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 451                         struct file_struct *file = flist->files[i];
 452                         recv_generator(local_name?local_name:f_name(file),
 453                                        flist,i,f);
 454                 }
 455
 456                 phase++;
 457                 if (verbose > 2)
 458                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 459
 460                 write_int(f,-1);
 461         }
 462 }