generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38
  39
  40 /* choose whether to skip a particular file */
  41 static int skip_file(char *fname,
  42                      struct file_struct *file, STRUCT_STAT *st)
  43 {
  44         if (st->st_size != file->length) {
  45                 return 0;
  46         }
  47
  48         /* if always checksum is set then we use the checksum instead
  49            of the file time to determine whether to sync */
  50         if (always_checksum && S_ISREG(st->st_mode)) {
  51                 char sum[MD4_SUM_LENGTH];
  52                 file_checksum(fname,sum,st->st_size);
  53                 if (remote_version < 21) {
  54                         return (memcmp(sum,file->sum,2) == 0);
  55                 } else {
  56                         return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
  57                 }
  58         }
  59
  60         if (size_only) {
  61                 return 1;
  62         }
  63
  64         if (ignore_times) {
  65                 return 0;
  66         }
  67
  68         return (st->st_mtime == file->modtime);
  69 }
  70
  71
  72 /* use a larger block size for really big files */
  73 static int adapt_block_size(struct file_struct *file, int bsize)
  74 {
  75         int ret;
  76
  77         if (bsize != BLOCK_SIZE) return bsize;
  78
  79         ret = file->length / (10000); /* rough heuristic */
  80         ret = ret & ~15; /* multiple of 16 */
  81         if (ret < bsize) ret = bsize;
  82         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  83         return ret;
  84 }
  85
  86
  87 /*
  88   send a sums struct down a fd
  89   */
  90 static void send_sums(struct sum_struct *s,int f_out)
  91 {
  92         int i;
  93
  94         /* tell the other guy how many we are going to be doing and how many
  95            bytes there are in the last chunk */
  96         write_int(f_out,s?s->count:0);
  97         write_int(f_out,s?s->n:block_size);
  98         write_int(f_out,s?s->remainder:0);
  99
 100         if (!s) return;
 101
 102         for (i=0;i<s->count;i++) {
 103                 write_int(f_out,s->sums[i].sum1);
 104                 write_buf(f_out,s->sums[i].sum2,csum_length);
 105         }
 106 }
 107
 108
 109 /*
 110   generate a stream of signatures/checksums that describe a buffer
 111
 112   generate approximately one checksum every n bytes
 113   */
 114 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 115 {
 116         int i;
 117         struct sum_struct *s;
 118         int count;
 119         int block_len = n;
 120         int remainder = (len%block_len);
 121         OFF_T offset = 0;
 122
 123         count = (len+(block_len-1))/block_len;
 124
 125         s = (struct sum_struct *)malloc(sizeof(*s));
 126         if (!s) out_of_memory("generate_sums");
 127
 128         s->count = count;
 129         s->remainder = remainder;
 130         s->n = n;
 131         s->flength = len;
 132
 133         if (count==0) {
 134                 s->sums = NULL;
 135                 return s;
 136         }
 137
 138         if (verbose > 3)
 139                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
 140                         s->count,s->remainder,s->n,(double)s->flength);
 141
 142         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 143         if (!s->sums) out_of_memory("generate_sums");
 144
 145         for (i=0;i<count;i++) {
 146                 int n1 = MIN(len,n);
 147                 char *map = map_ptr(buf,offset,n1);
 148
 149                 s->sums[i].sum1 = get_checksum1(map,n1);
 150                 get_checksum2(map,n1,s->sums[i].sum2);
 151
 152                 s->sums[i].offset = offset;
 153                 s->sums[i].len = n1;
 154                 s->sums[i].i = i;
 155
 156                 if (verbose > 3)
 157                         rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
 158                                 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 159
 160                 len -= n1;
 161                 offset += n1;
 162         }
 163
 164         return s;
 165 }
 166
 167
 168 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 169 {
 170         int fd;
 171         STRUCT_STAT st;
 172         struct map_struct *buf;
 173         struct sum_struct *s;
 174         int statret;
 175         struct file_struct *file = flist->files[i];
 176         char *fnamecmp;
 177         char fnamecmpbuf[MAXPATHLEN];
 178         extern char *compare_dest;
 179         extern int list_only;
 180         extern int preserve_perms;
 181         extern int only_existing;
 182
 183         if (list_only) return;
 184
 185         if (verbose > 2)
 186                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 187
 188         statret = link_stat(fname,&st);
 189
 190         if (only_existing && statret == -1 && errno == ENOENT) {
 191                 /* we only want to update existing files */
 192                 if (verbose > 1) rprintf(FINFO,"not creating %s\n",fname);
 193                 return;
 194         }
 195
 196         if (statret == 0 &&
 197             !preserve_perms &&
 198             (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
 199                 /* if the file exists already and we aren't perserving
 200                    presmissions then act as though the remote end sent
 201                    us the file permissions we already have */
 202                 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
 203         }
 204
 205         if (S_ISDIR(file->mode)) {
 206                 if (dry_run) return;
 207                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 208                         if (robust_unlink(fname) != 0) {
 209                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 210                                 return;
 211                         }
 212                         statret = -1;
 213                 }
 214                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 215                         if (!(relative_paths && errno==ENOENT &&
 216                               create_directory_path(fname)==0 &&
 217                               do_mkdir(fname,file->mode)==0)) {
 218                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 219                                         fname,strerror(errno));
 220                         }
 221                 }
 222                 if (set_perms(fname,file,NULL,0) && verbose)
 223                         rprintf(FINFO,"%s/\n",fname);
 224                 return;
 225         }
 226
 227         if (preserve_links && S_ISLNK(file->mode)) {
 228 #if SUPPORT_LINKS
 229                 char lnk[MAXPATHLEN];
 230                 int l;
 231                 extern int safe_symlinks;
 232
 233                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 234                         if (verbose) {
 235                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 236                                         fname,file->link);
 237                         }
 238                         return;
 239                 }
 240                 if (statret == 0) {
 241                         l = readlink(fname,lnk,MAXPATHLEN-1);
 242                         if (l > 0) {
 243                                 lnk[l] = 0;
 244                                 if (strcmp(lnk,file->link) == 0) {
 245                                         set_perms(fname,file,&st,1);
 246                                         return;
 247                                 }
 248                         }
 249                 }
 250                 delete_file(fname);
 251                 if (do_symlink(file->link,fname) != 0) {
 252                         rprintf(FERROR,"link %s -> %s : %s\n",
 253                                 fname,file->link,strerror(errno));
 254                 } else {
 255                         set_perms(fname,file,NULL,0);
 256                         if (verbose) {
 257                                 rprintf(FINFO,"%s -> %s\n",
 258                                         fname,file->link);
 259                         }
 260                 }
 261 #endif
 262                 return;
 263         }
 264
 265 #ifdef HAVE_MKNOD
 266         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 267                 if (statret != 0 ||
 268                     st.st_mode != file->mode ||
 269                     st.st_rdev != file->rdev) {
 270                         delete_file(fname);
 271                         if (verbose > 2)
 272                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 273                                         fname,(int)file->mode,(int)file->rdev);
 274                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 275                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 276                         } else {
 277                                 set_perms(fname,file,NULL,0);
 278                                 if (verbose)
 279                                         rprintf(FINFO,"%s\n",fname);
 280                         }
 281                 } else {
 282                         set_perms(fname,file,&st,1);
 283                 }
 284                 return;
 285         }
 286 #endif
 287
 288         if (preserve_hard_links && check_hard_link(file)) {
 289                 if (verbose > 1)
 290                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 291                 return;
 292         }
 293
 294         if (!S_ISREG(file->mode)) {
 295                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 296                 return;
 297         }
 298
 299         fnamecmp = fname;
 300
 301         if ((statret == -1) && (compare_dest != NULL)) {
 302                 /* try the file at compare_dest instead */
 303                 int saveerrno = errno;
 304                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 305                 statret = link_stat(fnamecmpbuf,&st);
 306                 if (!S_ISREG(st.st_mode))
 307                         statret = -1;
 308                 if (statret == -1)
 309                         errno = saveerrno;
 310                 else
 311                         fnamecmp = fnamecmpbuf;
 312         }
 313
 314         if (statret == -1) {
 315                 if (errno == ENOENT) {
 316                         write_int(f_out,i);
 317                         if (!dry_run) send_sums(NULL,f_out);
 318                 } else {
 319                         if (verbose > 1)
 320                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 321                 }
 322                 return;
 323         }
 324
 325         if (!S_ISREG(st.st_mode)) {
 326                 if (delete_file(fname) != 0) {
 327                         return;
 328                 }
 329
 330                 /* now pretend the file didn't exist */
 331                 write_int(f_out,i);
 332                 if (!dry_run) send_sums(NULL,f_out);
 333                 return;
 334         }
 335
 336         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 337                 if (verbose > 1)
 338                         rprintf(FINFO,"%s is newer\n",fname);
 339                 return;
 340         }
 341
 342         if (skip_file(fname, file, &st)) {
 343                 if (fnamecmp == fname)
 344                         set_perms(fname,file,&st,1);
 345                 return;
 346         }
 347
 348         if (dry_run) {
 349                 write_int(f_out,i);
 350                 return;
 351         }
 352
 353         if (whole_file) {
 354                 write_int(f_out,i);
 355                 send_sums(NULL,f_out);
 356                 return;
 357         }
 358
 359         /* open the file */
 360         fd = do_open(fnamecmp, O_RDONLY, 0);
 361
 362         if (fd == -1) {
 363                 rprintf(FERROR,"failed to open %s, continuing : %s\n",fnamecmp,strerror(errno));
 364                 /* pretend the file didn't exist */
 365                 write_int(f_out,i);
 366                 send_sums(NULL,f_out);
 367                 return;
 368         }
 369
 370         if (st.st_size > 0) {
 371                 buf = map_file(fd,st.st_size);
 372         } else {
 373                 buf = NULL;
 374         }
 375
 376         if (verbose > 3)
 377                 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
 378
 379         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 380
 381         if (verbose > 2)
 382                 rprintf(FINFO,"sending sums for %d\n",i);
 383
 384         write_int(f_out,i);
 385         send_sums(s,f_out);
 386
 387         close(fd);
 388         if (buf) unmap_file(buf);
 389
 390         free_sums(s);
 391 }
 392
 393
 394
 395 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 396 {
 397         int i;
 398         int phase=0;
 399
 400         if (verbose > 2)
 401                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 402                         (int)getpid(),flist->count);
 403
 404         for (i = 0; i < flist->count; i++) {
 405                 struct file_struct *file = flist->files[i];
 406                 mode_t saved_mode = file->mode;
 407                 if (!file->basename) continue;
 408
 409                 /* we need to ensure that any directories we create have writeable
 410                    permissions initially so that we can create the files within
 411                    them. This is then fixed after the files are transferred */
 412                 if (!am_root && S_ISDIR(file->mode)) {
 413                         file->mode |= S_IWUSR; /* user write */
 414                 }
 415
 416                 recv_generator(local_name?local_name:f_name(file),
 417                                flist,i,f);
 418
 419                 file->mode = saved_mode;
 420         }
 421
 422         phase++;
 423         csum_length = SUM_LENGTH;
 424         ignore_times=1;
 425
 426         if (verbose > 2)
 427                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 428
 429         write_int(f,-1);
 430
 431         /* we expect to just sit around now, so don't exit on a
 432            timeout. If we really get a timeout then the other process should
 433            exit */
 434         io_timeout = 0;
 435
 436         if (remote_version >= 13) {
 437                 /* in newer versions of the protocol the files can cycle through
 438                    the system more than once to catch initial checksum errors */
 439                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 440                         struct file_struct *file = flist->files[i];
 441                         recv_generator(local_name?local_name:f_name(file),
 442                                        flist,i,f);
 443                 }
 444
 445                 phase++;
 446                 if (verbose > 2)
 447                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 448
 449                 write_int(f,-1);
 450         }
 451 }