generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int io_timeout;
  35 extern int remote_version;
  36 extern int always_checksum;
  37
  38
  39 /* choose whether to skip a particular file */
  40 static int skip_file(char *fname,
  41                      struct file_struct *file, STRUCT_STAT *st)
  42 {
  43         if (st->st_size != file->length) {
  44                 return 0;
  45         }
  46
  47         /* if always checksum is set then we use the checksum instead
  48            of the file time to determine whether to sync */
  49         if (always_checksum && S_ISREG(st->st_mode)) {
  50                 char sum[MD4_SUM_LENGTH];
  51                 file_checksum(fname,sum,st->st_size);
  52                 return (memcmp(sum,file->sum,csum_length) == 0);
  53         }
  54
  55         if (ignore_times) {
  56                 return 0;
  57         }
  58
  59         return (st->st_mtime == file->modtime);
  60 }
  61
  62
  63 /* use a larger block size for really big files */
  64 static int adapt_block_size(struct file_struct *file, int bsize)
  65 {
  66         int ret;
  67
  68         if (bsize != BLOCK_SIZE) return bsize;
  69
  70         ret = file->length / (10000); /* rough heuristic */
  71         ret = ret & ~15; /* multiple of 16 */
  72         if (ret < bsize) ret = bsize;
  73         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  74         return ret;
  75 }
  76
  77
  78 /*
  79   send a sums struct down a fd
  80   */
  81 static void send_sums(struct sum_struct *s,int f_out)
  82 {
  83         int i;
  84
  85   /* tell the other guy how many we are going to be doing and how many
  86      bytes there are in the last chunk */
  87         write_int(f_out,s?s->count:0);
  88         write_int(f_out,s?s->n:block_size);
  89         write_int(f_out,s?s->remainder:0);
  90         if (s)
  91                 for (i=0;i<s->count;i++) {
  92                         write_int(f_out,s->sums[i].sum1);
  93                         write_buf(f_out,s->sums[i].sum2,csum_length);
  94                 }
  95 }
  96
  97
  98 /*
  99   generate a stream of signatures/checksums that describe a buffer
 100
 101   generate approximately one checksum every n bytes
 102   */
 103 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 104 {
 105         int i;
 106         struct sum_struct *s;
 107         int count;
 108         int block_len = n;
 109         int remainder = (len%block_len);
 110         OFF_T offset = 0;
 111
 112         count = (len+(block_len-1))/block_len;
 113
 114         s = (struct sum_struct *)malloc(sizeof(*s));
 115         if (!s) out_of_memory("generate_sums");
 116
 117         s->count = count;
 118         s->remainder = remainder;
 119         s->n = n;
 120         s->flength = len;
 121
 122         if (count==0) {
 123                 s->sums = NULL;
 124                 return s;
 125         }
 126
 127         if (verbose > 3)
 128                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 129                         s->count,s->remainder,s->n,(int)s->flength);
 130
 131         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 132         if (!s->sums) out_of_memory("generate_sums");
 133
 134         for (i=0;i<count;i++) {
 135                 int n1 = MIN(len,n);
 136                 char *map = map_ptr(buf,offset,n1);
 137
 138                 s->sums[i].sum1 = get_checksum1(map,n1);
 139                 get_checksum2(map,n1,s->sums[i].sum2);
 140
 141                 s->sums[i].offset = offset;
 142                 s->sums[i].len = n1;
 143                 s->sums[i].i = i;
 144
 145                 if (verbose > 3)
 146                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 147                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 148
 149                 len -= n1;
 150                 offset += n1;
 151         }
 152
 153         return s;
 154 }
 155
 156
 157 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 158 {
 159         int fd;
 160         STRUCT_STAT st;
 161         struct map_struct *buf;
 162         struct sum_struct *s;
 163         int statret;
 164         struct file_struct *file = flist->files[i];
 165         char *fnamecmp;
 166         char fnamecmpbuf[MAXPATHLEN];
 167         extern char *compare_dest;
 168
 169         if (verbose > 2)
 170                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 171
 172         statret = link_stat(fname,&st);
 173
 174         if (S_ISDIR(file->mode)) {
 175                 if (dry_run) return;
 176                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 177                         if (do_unlink(fname) != 0) {
 178                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 179                                 return;
 180                         }
 181                         statret = -1;
 182                 }
 183                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 184                         if (!(relative_paths && errno==ENOENT &&
 185                               create_directory_path(fname)==0 &&
 186                               do_mkdir(fname,file->mode)==0)) {
 187                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 188                                         fname,strerror(errno));
 189                         }
 190                 }
 191                 if (set_perms(fname,file,NULL,0) && verbose)
 192                         rprintf(FINFO,"%s/\n",fname);
 193                 return;
 194         }
 195
 196         if (preserve_links && S_ISLNK(file->mode)) {
 197 #if SUPPORT_LINKS
 198                 char lnk[MAXPATHLEN];
 199                 int l;
 200                 extern int safe_symlinks;
 201
 202                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 203                         if (verbose) {
 204                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 205                                         fname,file->link);
 206                         }
 207                         return;
 208                 }
 209                 if (statret == 0) {
 210                         l = readlink(fname,lnk,MAXPATHLEN-1);
 211                         if (l > 0) {
 212                                 lnk[l] = 0;
 213                                 if (strcmp(lnk,file->link) == 0) {
 214                                         set_perms(fname,file,&st,1);
 215                                         return;
 216                                 }
 217                         }
 218                 }
 219                 delete_file(fname);
 220                 if (do_symlink(file->link,fname) != 0) {
 221                         rprintf(FERROR,"link %s -> %s : %s\n",
 222                                 fname,file->link,strerror(errno));
 223                 } else {
 224                         set_perms(fname,file,NULL,0);
 225                         if (verbose) {
 226                                 rprintf(FINFO,"%s -> %s\n",
 227                                         fname,file->link);
 228                         }
 229                 }
 230 #endif
 231                 return;
 232         }
 233
 234 #ifdef HAVE_MKNOD
 235         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 236                 if (statret != 0 ||
 237                     st.st_mode != file->mode ||
 238                     st.st_rdev != file->rdev) {
 239                         delete_file(fname);
 240                         if (verbose > 2)
 241                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 242                                         fname,(int)file->mode,(int)file->rdev);
 243                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 244                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 245                         } else {
 246                                 set_perms(fname,file,NULL,0);
 247                                 if (verbose)
 248                                         rprintf(FINFO,"%s\n",fname);
 249                         }
 250                 } else {
 251                         set_perms(fname,file,&st,1);
 252                 }
 253                 return;
 254         }
 255 #endif
 256
 257         if (preserve_hard_links && check_hard_link(file)) {
 258                 if (verbose > 1)
 259                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 260                 return;
 261         }
 262
 263         if (!S_ISREG(file->mode)) {
 264                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 265                 return;
 266         }
 267
 268         fnamecmp = fname;
 269
 270         if ((statret == -1) && (compare_dest != NULL)) {
 271                 /* try the file at compare_dest instead */
 272                 int saveerrno = errno;
 273                 slprintf(fnamecmpbuf,MAXPATHLEN-1,"%s/%s",compare_dest,fname);
 274                 statret = link_stat(fnamecmpbuf,&st);
 275                 if (!S_ISREG(st.st_mode))
 276                         statret = -1;
 277                 if (statret == -1)
 278                         errno = saveerrno;
 279                 else
 280                         fnamecmp = fnamecmpbuf;
 281         }
 282
 283         if (statret == -1) {
 284                 if (errno == ENOENT) {
 285                         write_int(f_out,i);
 286                         if (!dry_run) send_sums(NULL,f_out);
 287                 } else {
 288                         if (verbose > 1)
 289                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 290                 }
 291                 return;
 292         }
 293
 294         if (!S_ISREG(st.st_mode)) {
 295                 if (delete_file(fname) != 0) {
 296                         return;
 297                 }
 298
 299                 /* now pretend the file didn't exist */
 300                 write_int(f_out,i);
 301                 if (!dry_run) send_sums(NULL,f_out);
 302                 return;
 303         }
 304
 305         if (update_only && st.st_mtime > file->modtime && fnamecmp == fname) {
 306                 if (verbose > 1)
 307                         rprintf(FINFO,"%s is newer\n",fname);
 308                 return;
 309         }
 310
 311         if (skip_file(fname, file, &st)) {
 312                 set_perms(fname,file,&st,1);
 313                 return;
 314         }
 315
 316         if (dry_run) {
 317                 write_int(f_out,i);
 318                 return;
 319         }
 320
 321         if (whole_file) {
 322                 write_int(f_out,i);
 323                 send_sums(NULL,f_out);
 324                 return;
 325         }
 326
 327         /* open the file */
 328         fd = open(fnamecmp,O_RDONLY);
 329
 330         if (fd == -1) {
 331                 rprintf(FERROR,"failed to open %s : %s\n",fnamecmp,strerror(errno));
 332                 rprintf(FERROR,"skipping %s\n",fname);
 333                 return;
 334         }
 335
 336         if (st.st_size > 0) {
 337                 buf = map_file(fd,st.st_size);
 338         } else {
 339                 buf = NULL;
 340         }
 341
 342         if (verbose > 3)
 343                 rprintf(FINFO,"gen mapped %s of size %d\n",fnamecmp,(int)st.st_size);
 344
 345         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 346
 347         if (verbose > 2)
 348                 rprintf(FINFO,"sending sums for %d\n",i);
 349
 350         write_int(f_out,i);
 351         send_sums(s,f_out);
 352
 353         close(fd);
 354         if (buf) unmap_file(buf);
 355
 356         free_sums(s);
 357 }
 358
 359
 360
 361 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 362 {
 363         int i;
 364         int phase=0;
 365
 366         if (verbose > 2)
 367                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 368                         (int)getpid(),flist->count);
 369
 370         for (i = 0; i < flist->count; i++) {
 371                 struct file_struct *file = flist->files[i];
 372                 mode_t saved_mode = file->mode;
 373                 if (!file->basename) continue;
 374
 375                 /* we need to ensure that any directories we create have writeable
 376                    permissions initially so that we can create the files within
 377                    them. This is then fixed after the files are transferred */
 378                 if (!am_root && S_ISDIR(file->mode)) {
 379                         file->mode |= S_IWUSR; /* user write */
 380                 }
 381
 382                 recv_generator(local_name?local_name:f_name(file),
 383                                flist,i,f);
 384
 385                 file->mode = saved_mode;
 386         }
 387
 388         phase++;
 389         csum_length = SUM_LENGTH;
 390         ignore_times=1;
 391
 392         if (verbose > 2)
 393                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 394
 395         write_int(f,-1);
 396
 397         /* we expect to just sit around now, so don't exit on a
 398            timeout. If we really get a timeout then the other process should
 399            exit */
 400         io_timeout = 0;
 401
 402         if (remote_version >= 13) {
 403                 /* in newer versions of the protocol the files can cycle through
 404                    the system more than once to catch initial checksum errors */
 405                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 406                         struct file_struct *file = flist->files[i];
 407                         recv_generator(local_name?local_name:f_name(file),
 408                                        flist,i,f);
 409                 }
 410
 411                 phase++;
 412                 if (verbose > 2)
 413                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 414
 415                 write_int(f,-1);
 416         }
 417 }