generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38 extern char *compare_dest;
  39
  40
  41 /* choose whether to skip a particular file */
  42 static int skip_file(char *fname,
  43                      struct file_struct *file, STRUCT_STAT *st)
  44 {
  45         if (st->st_size != file->length) {
  46                 return 0;
  47         }
  48
  49         /* if always checksum is set then we use the checksum instead
  50            of the file time to determine whether to sync */
  51         if (always_checksum && S_ISREG(st->st_mode)) {
  52                 char sum[MD4_SUM_LENGTH];
  53                 char fnamecmpdest[MAXPATHLEN];
  54
  55                 if (compare_dest != NULL) {
  56                         if (access(fname, 0) != 0) {
  57                                 slprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
  58                                                     compare_dest,fname);
  59                                 fname = fnamecmpdest;
  60                         }
  61                 }
  62                 file_checksum(fname,sum,st->st_size);
  63                 if (remote_version < 21) {
  64                         return (memcmp(sum,file->sum,2) == 0);
  65                 } else {
  66                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  67                 }
  68         }
  69
  70         if (size_only) {
  71                 return 1;
  72         }
  73
  74         if (ignore_times) {
  75                 return 0;
  76         }
  77
  78         return (st->st_mtime == file->modtime);
  79 }
  80
  81
  82 /* use a larger block size for really big files */
  83 static int adapt_block_size(struct file_struct *file, int bsize)
  84 {
  85         int ret;
  86
  87         if (bsize != BLOCK_SIZE) return bsize;
  88
  89         ret = file->length / (10000); /* rough heuristic */
  90         ret = ret & ~15; /* multiple of 16 */
  91         if (ret < bsize) ret = bsize;
  92         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  93         return ret;
  94 }
  95
  96
  97 /*
  98   send a sums struct down a fd
  99   */
 100 static void send_sums(struct sum_struct *s,int f_out)
 101 {
 102         int i;
 103
 104         /* tell the other guy how many we are going to be doing and how many
 105            bytes there are in the last chunk */
 106         write_int(f_out,s?s->count:0);
 107         write_int(f_out,s?s->n:block_size);
 108         write_int(f_out,s?s->remainder:0);
 109
 110         if (!s) return;
 111
 112         for (i=0;i<s->count;i++) {
 113                 write_int(f_out,s->sums[i].sum1);
 114                 write_buf(f_out,s->sums[i].sum2,csum_length);
 115         }
 116 }
 117
 118
 119 /*
 120   generate a stream of signatures/checksums that describe a buffer
 121
 122   generate approximately one checksum every n bytes
 123   */
 124 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 125 {
 126         int i;
 127         struct sum_struct *s;
 128         int count;
 129         int block_len = n;
 130         int remainder = (len%block_len);
 131         OFF_T offset = 0;
 132
 133         count = (len+(block_len-1))/block_len;
 134
 135         s = (struct sum_struct *)malloc(sizeof(*s));
 136         if (!s) out_of_memory("generate_sums");
 137
 138         s->count = count;
 139         s->remainder = remainder;
 140         s->n = n;
 141         s->flength = len;
 142
 143         if (count==0) {
 144                 s->sums = NULL;
 145                 return s;
 146         }
 147
 148         if (verbose > 3)
 149                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 150                         s->count,s->remainder,s->n,(double)s->flength);
 151
 152         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 153         if (!s->sums) out_of_memory("generate_sums");
 154
 155         for (i=0;i<count;i++) {
 156                 int n1 = MIN(len,n);
 157                 char *map = map_ptr(buf,offset,n1);
 158
 159                 s->sums[i].sum1 = get_checksum1(map,n1);
 160                 get_checksum2(map,n1,s->sums[i].sum2);
 161
 162                 s->sums[i].offset = offset;
 163                 s->sums[i].len = n1;
 164                 s->sums[i].i = i;
 165
 166                 if (verbose > 3)
 167                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 168                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 169
 170                 len -= n1;
 171                 offset += n1;
 172         }
 173
 174         return s;
 175 }
 176
 177
 178 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 179 {
 180         int fd;
 181         STRUCT_STAT st;
 182         struct map_struct *buf;
 183         struct sum_struct *s;
 184         int statret;
 185         struct file_struct *file = flist->files[i];
 186         char *fnamecmp;
 187         char fnamecmpbuf[MAXPATHLEN];
 188         extern char *compare_dest;
 189         extern int list_only;
 190         extern int preserve_perms;
 191         extern int only_existing;
 192
 193         if (list_only) return;
 194
 195         if (verbose > 2)
 196                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 197
 198         statret = link_stat(fname,&st);
 199
 200         if (only_existing && statret == -1 && errno == ENOENT) {
 201                 /* we only want to update existing files */
 202                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 203                 return;
 204         }
 205
 206         if (statret == 0 &&
 207             !preserve_perms &&
 208             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 209                 /* if the file exists already and we aren't perserving
 210                    presmissions then act as though the remote end sent
 211                    us the file permissions we already have */
 212                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 213         }
 214
 215         if (S_ISDIR(file->mode)) {
 216                 if (dry_run) return;
 217                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 218                         if (robust_unlink(fname) != 0) {
 219                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 220                                 return;
 221                         }
 222                         statret = -1;
 223                 }
 224                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 225                         if (!(relative_paths && errno==ENOENT &&
 226                               create_directory_path(fname)==0 &&
 227                               do_mkdir(fname,file->mode)==0)) {
 228                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 229                                         fname,strerror(errno));
 230                         }
 231                 }
 232                 if (set_perms(fname,file,NULL,0) && verbose)
 233                         rprintf(FINFO,"%s/\n",fname);
 234                 return;
 235         }
 236
 237         if (preserve_links && S_ISLNK(file->mode)) {
 238 #if SUPPORT_LINKS
 239                 char lnk[MAXPATHLEN];
 240                 int l;
 241                 extern int safe_symlinks;
 242
 243                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 244                         if (verbose) {
 245                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 246                                         fname,file->link);
 247                         }
 248                         return;
 249                 }
 250                 if (statret == 0) {
 251                         l = readlink(fname,lnk,MAXPATHLEN-1);
 252                         if (l > 0) {
 253                                 lnk[l] = 0;
 254                                 if (strcmp(lnk,file->link) == 0) {
 255                                         set_perms(fname,file,&st,1);
 256                                         return;
 257                                 }
 258                         }
 259                         delete_file(fname);
 260                 }
 261                 if (do_symlink(file->link,fname) != 0) {
 262                         rprintf(FERROR,"symlink %s -> %s : %s\n",
 263                                 fname,file->link,strerror(errno));
 264                 } else {
 265                         set_perms(fname,file,NULL,0);
 266                         if (verbose) {
 267                                 rprintf(FINFO,"%s -> %s\n",
 268                                         fname,file->link);
 269                         }
 270                 }
 271 #endif
 272                 return;
 273         }
 274
 275 #ifdef HAVE_MKNOD
 276         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 277                 if (statret != 0 ||
 278                     st.st_mode != file->mode ||
 279                     st.st_rdev != file->rdev) {
 280                         delete_file(fname);
 281                         if (verbose > 2)
 282                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 283                                         fname,(int)file->mode,(int)file->rdev);
 284                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 285                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 286                         } else {
 287                                 set_perms(fname,file,NULL,0);
 288                                 if (verbose)
 289                                         rprintf(FINFO,"%s\n",fname);
 290                         }
 291                 } else {
 292                         set_perms(fname,file,&st,1);
 293                 }
 294                 return;
 295         }
 296 #endif
 297
 298         if (preserve_hard_links && check_hard_link(file)) {
 299                 if (verbose > 1)
 300                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 301                 return;
 302         }
 303
 304         if (!S_ISREG(file->mode)) {
 305                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 306                 return;
 307         }
 308
 309         fnamecmp = fname;
 310
 311         if ((statret == -1) && (compare_dest != NULL)) {
 312                 /* try the file at compare_dest instead */
 313                 int saveerrno = errno;
 314                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 315                 statret = link_stat(fnamecmpbuf,&st);
 316                 if (!S_ISREG(st.st_mode))
 317                         statret = -1;
 318                 if (statret == -1)
 319                         errno = saveerrno;
 320                 else
 321                         fnamecmp = fnamecmpbuf;
 322         }
 323
 324         if (statret == -1) {
 325                 if (errno == ENOENT) {
 326                         write_int(f_out,i);
 327                         if (!dry_run) send_sums(NULL,f_out);
 328                 } else {
 329                         if (verbose > 1)
 330                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 331                 }
 332                 return;
 333         }
 334
 335         if (!S_ISREG(st.st_mode)) {
 336                 if (delete_file(fname) != 0) {
 337                         return;
 338                 }
 339
 340                 /* now pretend the file didn't exist */
 341                 write_int(f_out,i);
 342                 if (!dry_run) send_sums(NULL,f_out);
 343                 return;
 344         }
 345
 346         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 347                 if (verbose > 1)
 348                         rprintf(FINFO,"%s is newer\n",fname);
 349                 return;
 350         }
 351
 352         if (skip_file(fname, file, &st)) {
 353                 if (fnamecmp == fname)
 354                         set_perms(fname,file,&st,1);
 355                 return;
 356         }
 357
 358         if (dry_run) {
 359                 write_int(f_out,i);
 360                 return;
 361         }
 362
 363         if (whole_file) {
 364                 write_int(f_out,i);
 365                 send_sums(NULL,f_out);
 366                 return;
 367         }
 368
 369         /* open the file */
 370         fd = do_open(fnamecmp, O_RDONLY, 0);
 371
 372         if (fd == -1) {
 373                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 374                 /* pretend the file didn't exist */
 375                 write_int(f_out,i);
 376                 send_sums(NULL,f_out);
 377                 return;
 378         }
 379
 380         if (st.st_size > 0) {
 381                 buf = map_file(fd,st.st_size);
 382         } else {
 383                 buf = NULL;
 384         }
 385
 386         if (verbose > 3)
 387                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 388
 389         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 390
 391         if (verbose > 2)
 392                 rprintf(FINFO,"sending sums for %d\n",i);
 393
 394         write_int(f_out,i);
 395         send_sums(s,f_out);
 396
 397         close(fd);
 398         if (buf) unmap_file(buf);
 399
 400         free_sums(s);
 401 }
 402
 403
 404
 405 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 406 {
 407         int i;
 408         int phase=0;
 409
 410         if (verbose > 2)
 411                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 412                         (int)getpid(),flist->count);
 413
 414         for (i = 0; i < flist->count; i++) {
 415                 struct file_struct *file = flist->files[i];
 416                 mode_t saved_mode = file->mode;
 417                 if (!file->basename) continue;
 418
 419                 /* we need to ensure that any directories we create have writeable
 420                    permissions initially so that we can create the files within
 421                    them. This is then fixed after the files are transferred */
 422                 if (!am_root && S_ISDIR(file->mode)) {
 423                         file->mode |= S_IWUSR; /* user write */
 424                 }
 425
 426                 recv_generator(local_name?local_name:f_name(file),
 427                                flist,i,f);
 428
 429                 file->mode = saved_mode;
 430         }
 431
 432         phase++;
 433         csum_length = SUM_LENGTH;
 434         ignore_times=1;
 435
 436         if (verbose > 2)
 437                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 438
 439         write_int(f,-1);
 440
 441         /* we expect to just sit around now, so don't exit on a
 442            timeout. If we really get a timeout then the other process should
 443            exit */
 444         io_timeout = 0;
 445
 446         if (remote_version >= 13) {
 447                 /* in newer versions of the protocol the files can cycle through
 448                    the system more than once to catch initial checksum errors */
 449                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 450                         struct file_struct *file = flist->files[i];
 451                         recv_generator(local_name?local_name:f_name(file),
 452                                        flist,i,f);
 453                 }
 454
 455                 phase++;
 456                 if (verbose > 2)
 457                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 458
 459                 write_int(f,-1);
 460         }
 461 }