generator.c

   1 /*
   2    Copyright (C) Andrew Tridgell 1996
   3    Copyright (C) Paul Mackerras 1996
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, write to the Free Software
  17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20 #include "rsync.h"
  21
  22 extern int verbose;
  23 extern int dry_run;
  24 extern int relative_paths;
  25 extern int preserve_links;
  26 extern int am_root;
  27 extern int preserve_devices;
  28 extern int preserve_hard_links;
  29 extern int update_only;
  30 extern int whole_file;
  31 extern int block_size;
  32 extern int csum_length;
  33 extern int ignore_times;
  34 extern int io_timeout;
  35 extern int remote_version;
  36 extern int always_checksum;
  37
  38
  39 /* choose whether to skip a particular file */
  40 static int skip_file(char *fname,
  41                      struct file_struct *file, STRUCT_STAT *st)
  42 {
  43         if (st->st_size != file->length) {
  44                 return 0;
  45         }
  46
  47         /* if always checksum is set then we use the checksum instead
  48            of the file time to determine whether to sync */
  49         if (always_checksum && S_ISREG(st->st_mode)) {
  50                 char sum[MD4_SUM_LENGTH];
  51                 file_checksum(fname,sum,st->st_size);
  52                 return (memcmp(sum,file->sum,csum_length) == 0);
  53         }
  54
  55         if (ignore_times) {
  56                 return 0;
  57         }
  58
  59         return (st->st_mtime == file->modtime);
  60 }
  61
  62
  63 /* use a larger block size for really big files */
  64 static int adapt_block_size(struct file_struct *file, int bsize)
  65 {
  66         int ret;
  67
  68         if (bsize != BLOCK_SIZE) return bsize;
  69
  70         ret = file->length / (10000); /* rough heuristic */
  71         ret = ret & ~15; /* multiple of 16 */
  72         if (ret < bsize) ret = bsize;
  73         if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
  74         return ret;
  75 }
  76
  77
  78 /*
  79   send a sums struct down a fd
  80   */
  81 static void send_sums(struct sum_struct *s,int f_out)
  82 {
  83         int i;
  84
  85   /* tell the other guy how many we are going to be doing and how many
  86      bytes there are in the last chunk */
  87         write_int(f_out,s?s->count:0);
  88         write_int(f_out,s?s->n:block_size);
  89         write_int(f_out,s?s->remainder:0);
  90         if (s)
  91                 for (i=0;i<s->count;i++) {
  92                         write_int(f_out,s->sums[i].sum1);
  93                         write_buf(f_out,s->sums[i].sum2,csum_length);
  94                 }
  95 }
  96
  97
  98 /*
  99   generate a stream of signatures/checksums that describe a buffer
 100
 101   generate approximately one checksum every n bytes
 102   */
 103 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
 104 {
 105         int i;
 106         struct sum_struct *s;
 107         int count;
 108         int block_len = n;
 109         int remainder = (len%block_len);
 110         OFF_T offset = 0;
 111
 112         count = (len+(block_len-1))/block_len;
 113
 114         s = (struct sum_struct *)malloc(sizeof(*s));
 115         if (!s) out_of_memory("generate_sums");
 116
 117         s->count = count;
 118         s->remainder = remainder;
 119         s->n = n;
 120         s->flength = len;
 121
 122         if (count==0) {
 123                 s->sums = NULL;
 124                 return s;
 125         }
 126
 127         if (verbose > 3)
 128                 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
 129                         s->count,s->remainder,s->n,(int)s->flength);
 130
 131         s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
 132         if (!s->sums) out_of_memory("generate_sums");
 133
 134         for (i=0;i<count;i++) {
 135                 int n1 = MIN(len,n);
 136                 char *map = map_ptr(buf,offset,n1);
 137
 138                 s->sums[i].sum1 = get_checksum1(map,n1);
 139                 get_checksum2(map,n1,s->sums[i].sum2);
 140
 141                 s->sums[i].offset = offset;
 142                 s->sums[i].len = n1;
 143                 s->sums[i].i = i;
 144
 145                 if (verbose > 3)
 146                         rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
 147                                 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
 148
 149                 len -= n1;
 150                 offset += n1;
 151         }
 152
 153         return s;
 154 }
 155
 156
 157 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 158 {
 159         int fd;
 160         STRUCT_STAT st;
 161         struct map_struct *buf;
 162         struct sum_struct *s;
 163         int statret;
 164         struct file_struct *file = flist->files[i];
 165
 166         if (verbose > 2)
 167                 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
 168
 169         statret = link_stat(fname,&st);
 170
 171         if (S_ISDIR(file->mode)) {
 172                 if (dry_run) return;
 173                 if (statret == 0 && !S_ISDIR(st.st_mode)) {
 174                         if (do_unlink(fname) != 0) {
 175                                 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
 176                                 return;
 177                         }
 178                         statret = -1;
 179                 }
 180                 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
 181                         if (!(relative_paths && errno==ENOENT &&
 182                               create_directory_path(fname)==0 &&
 183                               do_mkdir(fname,file->mode)==0)) {
 184                                 rprintf(FERROR,"mkdir %s : %s (2)\n",
 185                                         fname,strerror(errno));
 186                         }
 187                 }
 188                 if (set_perms(fname,file,NULL,0) && verbose)
 189                         rprintf(FINFO,"%s/\n",fname);
 190                 return;
 191         }
 192
 193         if (preserve_links && S_ISLNK(file->mode)) {
 194 #if SUPPORT_LINKS
 195                 char lnk[MAXPATHLEN];
 196                 int l;
 197                 extern int safe_symlinks;
 198
 199                 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
 200                         if (verbose) {
 201                                 rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
 202                                         fname,file->link);
 203                         }
 204                         return;
 205                 }
 206                 if (statret == 0) {
 207                         l = readlink(fname,lnk,MAXPATHLEN-1);
 208                         if (l > 0) {
 209                                 lnk[l] = 0;
 210                                 if (strcmp(lnk,file->link) == 0) {
 211                                         set_perms(fname,file,&st,1);
 212                                         return;
 213                                 }
 214                         }
 215                 }
 216                 delete_file(fname);
 217                 if (do_symlink(file->link,fname) != 0) {
 218                         rprintf(FERROR,"link %s -> %s : %s\n",
 219                                 fname,file->link,strerror(errno));
 220                 } else {
 221                         set_perms(fname,file,NULL,0);
 222                         if (verbose) {
 223                                 rprintf(FINFO,"%s -> %s\n",
 224                                         fname,file->link);
 225                         }
 226                 }
 227 #endif
 228                 return;
 229         }
 230
 231 #ifdef HAVE_MKNOD
 232         if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
 233                 if (statret != 0 ||
 234                     st.st_mode != file->mode ||
 235                     st.st_rdev != file->rdev) {
 236                         delete_file(fname);
 237                         if (verbose > 2)
 238                                 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
 239                                         fname,(int)file->mode,(int)file->rdev);
 240                         if (do_mknod(fname,file->mode,file->rdev) != 0) {
 241                                 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
 242                         } else {
 243                                 set_perms(fname,file,NULL,0);
 244                                 if (verbose)
 245                                         rprintf(FINFO,"%s\n",fname);
 246                         }
 247                 } else {
 248                         set_perms(fname,file,&st,1);
 249                 }
 250                 return;
 251         }
 252 #endif
 253
 254         if (preserve_hard_links && check_hard_link(file)) {
 255                 if (verbose > 1)
 256                         rprintf(FINFO,"%s is a hard link\n",f_name(file));
 257                 return;
 258         }
 259
 260         if (!S_ISREG(file->mode)) {
 261                 rprintf(FINFO,"skipping non-regular file %s\n",fname);
 262                 return;
 263         }
 264
 265         if (statret == -1) {
 266                 if (errno == ENOENT) {
 267                         write_int(f_out,i);
 268                         if (!dry_run) send_sums(NULL,f_out);
 269                 } else {
 270                         if (verbose > 1)
 271                                 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
 272                 }
 273                 return;
 274         }
 275
 276         if (!S_ISREG(st.st_mode)) {
 277                 if (delete_file(fname) != 0) {
 278                         return;
 279                 }
 280
 281                 /* now pretend the file didn't exist */
 282                 write_int(f_out,i);
 283                 if (!dry_run) send_sums(NULL,f_out);
 284                 return;
 285         }
 286
 287         if (update_only && st.st_mtime > file->modtime) {
 288                 if (verbose > 1)
 289                         rprintf(FINFO,"%s is newer\n",fname);
 290                 return;
 291         }
 292
 293         if (skip_file(fname, file, &st)) {
 294                 set_perms(fname,file,&st,1);
 295                 return;
 296         }
 297
 298         if (dry_run) {
 299                 write_int(f_out,i);
 300                 return;
 301         }
 302
 303         if (whole_file) {
 304                 write_int(f_out,i);
 305                 send_sums(NULL,f_out);
 306                 return;
 307         }
 308
 309         /* open the file */
 310         fd = open(fname,O_RDONLY);
 311
 312         if (fd == -1) {
 313                 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
 314                 rprintf(FERROR,"skipping %s\n",fname);
 315                 return;
 316         }
 317
 318         if (st.st_size > 0) {
 319                 buf = map_file(fd,st.st_size);
 320         } else {
 321                 buf = NULL;
 322         }
 323
 324         if (verbose > 3)
 325                 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
 326
 327         s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
 328
 329         if (verbose > 2)
 330                 rprintf(FINFO,"sending sums for %d\n",i);
 331
 332         write_int(f_out,i);
 333         send_sums(s,f_out);
 334
 335         close(fd);
 336         if (buf) unmap_file(buf);
 337
 338         free_sums(s);
 339 }
 340
 341
 342
 343 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 344 {
 345         int i;
 346         int phase=0;
 347
 348         if (verbose > 2)
 349                 rprintf(FINFO,"generator starting pid=%d count=%d\n",
 350                         (int)getpid(),flist->count);
 351
 352         for (i = 0; i < flist->count; i++) {
 353                 struct file_struct *file = flist->files[i];
 354                 mode_t saved_mode = file->mode;
 355                 if (!file->basename) continue;
 356
 357                 /* we need to ensure that any directories we create have writeable
 358                    permissions initially so that we can create the files within
 359                    them. This is then fixed after the files are transferred */
 360                 if (!am_root && S_ISDIR(file->mode)) {
 361                         file->mode |= S_IWUSR; /* user write */
 362                 }
 363
 364                 recv_generator(local_name?local_name:f_name(file),
 365                                flist,i,f);
 366
 367                 file->mode = saved_mode;
 368         }
 369
 370         phase++;
 371         csum_length = SUM_LENGTH;
 372         ignore_times=1;
 373
 374         if (verbose > 2)
 375                 rprintf(FINFO,"generate_files phase=%d\n",phase);
 376
 377         write_int(f,-1);
 378
 379         /* we expect to just sit around now, so don't exit on a
 380            timeout. If we really get a timeout then the other process should
 381            exit */
 382         io_timeout = 0;
 383
 384         if (remote_version >= 13) {
 385                 /* in newer versions of the protocol the files can cycle through
 386                    the system more than once to catch initial checksum errors */
 387                 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
 388                         struct file_struct *file = flist->files[i];
 389                         recv_generator(local_name?local_name:f_name(file),
 390                                        flist,i,f);
 391                 }
 392
 393                 phase++;
 394                 if (verbose > 2)
 395                         rprintf(FINFO,"generate_files phase=%d\n",phase);
 396
 397                 write_int(f,-1);
 398         }
 399 }