generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int size_only;
  35 extern int io_timeout;
  36 extern int remote_version;
  37 extern int always_checksum;
  38
  39
  40 /* choose whether to skip a particular file */
  41 static int skip_file(char *fname,
  42                      struct file_struct *file, STRUCT_STAT *st)
  43 {
  44         if (st->st_size != file->length) {
  45                 return 0;
  46         }
  47
  48         /* if always checksum is set then we use the checksum instead
  49            of the file time to determine whether to sync */
  50         if (always_checksum && S_ISREG(st->st_mode)) {
  51                 char sum[MD4_SUM_LENGTH];
  52                 file_checksum(fname,sum,st->st_size);
  53                 return (memcmp(sum,file->sum,csum_length) == 0);
  54         }
  55
  56         if (size_only) {
  57                 return 1;
  58         }
  59
  60         if (ignore_times) {
  61                 return 0;
  62         }
  63
  64         return (st->st_mtime == file->modtime);
  65 }
  66
  67
  68 /* use a larger block size for really big files */
  69 static int adapt_block_size(struct file_struct *file, int bsize)
  70 {
  71         int ret;
  72
  73         if (bsize != BLOCK_SIZE) return bsize;
  74
  75         ret = file->length / (10000); /* rough heuristic */
  76         ret = ret & ~15; /* multiple of 16 */
  77         if (ret < bsize) ret = bsize;
  78         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  79         return ret;
  80 }
  81
  82
  83 /*
  84   send a sums struct down a fd
  85   */
  86 static void send_sums(struct sum_struct *s,int f_out)
  87 {
  88         int i;
  89
  90   /* tell the other guy how many we are going to be doing and how many
  91      bytes there are in the last chunk */
  92         write_int(f_out,s?s->count:0);
  93         write_int(f_out,s?s->n:block_size);
  94         write_int(f_out,s?s->remainder:0);
  95         if (s)
  96                 for (i=0;i<s->count;i++) {
  97                         write_int(f_out,s->sums[i].sum1);
  98                         write_buf(f_out,s->sums[i].sum2,csum_length);
  99                 }
 100 }
 101
 102
 103 /*
 104   generate a stream of signatures/checksums that describe a buffer
 105
 106   generate approximately one checksum every n bytes
 107   */
 108 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 109 {
 110         int i;
 111         struct sum_struct *s;
 112         int count;
 113         int block_len = n;
 114         int remainder = (len%block_len);
 115         OFF_T offset = 0;
 116
 117         count = (len+(block_len-1))/block_len;
 118
 119         s = (struct sum_struct *)malloc(sizeof(*s));
 120         if (!s) out_of_memory("generate_sums");
 121
 122         s->count = count;
 123         s->remainder = remainder;
 124         s->n = n;
 125         s->flength = len;
 126
 127         if (count==0) {
 128                 s->sums = NULL;
 129                 return s;
 130         }
 131
 132         if (verbose > 3)
 133                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 134                         s->count,s->remainder,s->n,(int)s->flength);
 135
 136         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 137         if (!s->sums) out_of_memory("generate_sums");
 138
 139         for (i=0;i<count;i++) {
 140                 int n1 = MIN(len,n);
 141                 char *map = map_ptr(buf,offset,n1);
 142
 143                 s->sums[i].sum1 = get_checksum1(map,n1);
 144                 get_checksum2(map,n1,s->sums[i].sum2);
 145
 146                 s->sums[i].offset = offset;
 147                 s->sums[i].len = n1;
 148                 s->sums[i].i = i;
 149
 150                 if (verbose > 3)
 151                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 152                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 153
 154                 len -= n1;
 155                 offset += n1;
 156         }
 157
 158         return s;
 159 }
 160
 161
 162 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 163 {
 164         int fd;
 165         STRUCT_STAT st;
 166         struct map_struct *buf;
 167         struct sum_struct *s;
 168         int statret;
 169         struct file_struct *file = flist->files[i];
 170         char *fnamecmp;
 171         char fnamecmpbuf[MAXPATHLEN];
 172         extern char *compare_dest;
 173         extern int list_only;
 174
 175         if (list_only) return;
 176
 177         if (verbose > 2)
 178                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 179
 180         statret = link_stat(fname,&st);
 181
 182         if (S_ISDIR(file->mode)) {
 183                 if (dry_run) return;
 184                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 185                         if (robust_unlink(fname) != 0) {
 186                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 187                                 return;
 188                         }
 189                         statret = -1;
 190                 }
 191                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 192                         if (!(relative_paths && errno==ENOENT &&
 193                               create_directory_path(fname)==0 &&
 194                               do_mkdir(fname,file->mode)==0)) {
 195                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 196                                         fname,strerror(errno));
 197                         }
 198                 }
 199                 if (set_perms(fname,file,NULL,0) && verbose)
 200                         rprintf(FINFO,"%s/\n",fname);
 201                 return;
 202         }
 203
 204         if (preserve_links && S_ISLNK(file->mode)) {
 205 #if SUPPORT_LINKS
 206                 char lnk[MAXPATHLEN];
 207                 int l;
 208                 extern int safe_symlinks;
 209
 210                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 211                         if (verbose) {
 212                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 213                                         fname,file->link);
 214                         }
 215                         return;
 216                 }
 217                 if (statret == 0) {
 218                         l = readlink(fname,lnk,MAXPATHLEN-1);
 219                         if (l > 0) {
 220                                 lnk[l] = 0;
 221                                 if (strcmp(lnk,file->link) == 0) {
 222                                         set_perms(fname,file,&st,1);
 223                                         return;
 224                                 }
 225                         }
 226                 }
 227                 delete_file(fname);
 228                 if (do_symlink(file->link,fname) != 0) {
 229                         rprintf(FERROR,"link %s -> %s : %s\n",
 230                                 fname,file->link,strerror(errno));
 231                 } else {
 232                         set_perms(fname,file,NULL,0);
 233                         if (verbose) {
 234                                 rprintf(FINFO,"%s -> %s\n",
 235                                         fname,file->link);
 236                         }
 237                 }
 238 #endif
 239                 return;
 240         }
 241
 242 #ifdef HAVE_MKNOD
 243         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 244                 if (statret != 0 ||
 245                     st.st_mode != file->mode ||
 246                     st.st_rdev != file->rdev) {
 247                         delete_file(fname);
 248                         if (verbose > 2)
 249                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 250                                         fname,(int)file->mode,(int)file->rdev);
 251                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 252                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 253                         } else {
 254                                 set_perms(fname,file,NULL,0);
 255                                 if (verbose)
 256                                         rprintf(FINFO,"%s\n",fname);
 257                         }
 258                 } else {
 259                         set_perms(fname,file,&st,1);
 260                 }
 261                 return;
 262         }
 263 #endif
 264
 265         if (preserve_hard_links && check_hard_link(file)) {
 266                 if (verbose > 1)
 267                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 268                 return;
 269         }
 270
 271         if (!S_ISREG(file->mode)) {
 272                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 273                 return;
 274         }
 275
 276         fnamecmp = fname;
 277
 278         if ((statret == -1) && (compare_dest != NULL)) {
 279                 /* try the file at compare_dest instead */
 280                 int saveerrno = errno;
 281                 slprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
 282                 statret = link_stat(fnamecmpbuf,&st);
 283                 if (!S_ISREG(st.st_mode))
 284                         statret = -1;
 285                 if (statret == -1)
 286                         errno = saveerrno;
 287                 else
 288                         fnamecmp = fnamecmpbuf;
 289         }
 290
 291         if (statret == -1) {
 292                 if (errno == ENOENT) {
 293                         write_int(f_out,i);
 294                         if (!dry_run) send_sums(NULL,f_out);
 295                 } else {
 296                         if (verbose > 1)
 297                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 298                 }
 299                 return;
 300         }
 301
 302         if (!S_ISREG(st.st_mode)) {
 303                 if (delete_file(fname) != 0) {
 304                         return;
 305                 }
 306
 307                 /* now pretend the file didn't exist */
 308                 write_int(f_out,i);
 309                 if (!dry_run) send_sums(NULL,f_out);
 310                 return;
 311         }
 312
 313         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 314                 if (verbose > 1)
 315                         rprintf(FINFO,"%s is newer\n",fname);
 316                 return;
 317         }
 318
 319         if (skip_file(fname, file, &st)) {
 320                 if (fnamecmp == fname)
 321                         set_perms(fname,file,&st,1);
 322                 return;
 323         }
 324
 325         if (dry_run) {
 326                 write_int(f_out,i);
 327                 return;
 328         }
 329
 330         if (whole_file) {
 331                 write_int(f_out,i);
 332                 send_sums(NULL,f_out);
 333                 return;
 334         }
 335
 336         /* open the file */
 337         fd = open(fnamecmp,O_RDONLY);
 338
 339         if (fd == -1) {
 340                 rprintf(FERROR,"failed to open %s : %s\n",fnamecmp,strerror(errno));
 341                 rprintf(FERROR,"skipping %s\n",fname);
 342                 return;
 343         }
 344
 345         if (st.st_size > 0) {
 346                 buf = map_file(fd,st.st_size);
 347         } else {
 348                 buf = NULL;
 349         }
 350
 351         if (verbose > 3)
 352                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 353
 354         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 355
 356         if (verbose > 2)
 357                 rprintf(FINFO,"sending sums for %d\n",i);
 358
 359         write_int(f_out,i);
 360         send_sums(s,f_out);
 361
 362         close(fd);
 363         if (buf) unmap_file(buf);
 364
 365         free_sums(s);
 366 }
 367
 368
 369
 370 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 371 {
 372         int i;
 373         int phase=0;
 374
 375         if (verbose > 2)
 376                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 377                         (int)getpid(),flist->count);
 378
 379         for (i = 0; i < flist->count; i++) {
 380                 struct file_struct *file = flist->files[i];
 381                 mode_t saved_mode = file->mode;
 382                 if (!file->basename) continue;
 383
 384                 /* we need to ensure that any directories we create have writeable
 385                    permissions initially so that we can create the files within
 386                    them. This is then fixed after the files are transferred */
 387                 if (!am_root && S_ISDIR(file->mode)) {
 388                         file->mode |= S_IWUSR; /* user write */
 389                 }
 390
 391                 recv_generator(local_name?local_name:f_name(file),
 392                                flist,i,f);
 393
 394                 file->mode = saved_mode;
 395         }
 396
 397         phase++;
 398         csum_length = SUM_LENGTH;
 399         ignore_times=1;
 400
 401         if (verbose > 2)
 402                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 403
 404         write_int(f,-1);
 405
 406         /* we expect to just sit around now, so don't exit on a
 407            timeout. If we really get a timeout then the other process should
 408            exit */
 409         io_timeout = 0;
 410
 411         if (remote_version >= 13) {
 412                 /* in newer versions of the protocol the files can cycle through
 413                    the system more than once to catch initial checksum errors */
 414                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 415                         struct file_struct *file = flist->files[i];
 416                         recv_generator(local_name?local_name:f_name(file),
 417                                        flist,i,f);
 418                 }
 419
 420                 phase++;
 421                 if (verbose > 2)
 422                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 423
 424                 write_int(f,-1);
 425         }
 426 }