Apparently SGI make doesn't like $< in non-implicit rules.
[rsync/rsync.git] / generator.c
... / ...
CommitLineData
1/* -*- c-file-style: "linux" -*-
2
3 rsync -- fast file replication program
4
5 Copyright (C) 1996-2000 by Andrew Tridgell
6 Copyright (C) Paul Mackerras 1996
7 Copyright (C) 2002 by Martin Pool <mbp@samba.org>
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22*/
23
24#include "rsync.h"
25
26extern int verbose;
27extern int dry_run;
28extern int relative_paths;
29extern int preserve_links;
30extern int am_root;
31extern int preserve_devices;
32extern int preserve_hard_links;
33extern int update_only;
34extern int opt_ignore_existing;
35extern int block_size;
36extern int csum_length;
37extern int ignore_times;
38extern int size_only;
39extern int io_timeout;
40extern int remote_version;
41extern int always_checksum;
42extern int modify_window;
43extern char *compare_dest;
44
45
46/* choose whether to skip a particular file */
47static int skip_file(char *fname,
48 struct file_struct *file, STRUCT_STAT *st)
49{
50 if (st->st_size != file->length) {
51 return 0;
52 }
53
54 /* if always checksum is set then we use the checksum instead
55 of the file time to determine whether to sync */
56 if (always_checksum && S_ISREG(st->st_mode)) {
57 char sum[MD4_SUM_LENGTH];
58 char fnamecmpdest[MAXPATHLEN];
59
60 if (compare_dest != NULL) {
61 if (access(fname, 0) != 0) {
62 snprintf(fnamecmpdest,MAXPATHLEN,"%s/%s",
63 compare_dest,fname);
64 fname = fnamecmpdest;
65 }
66 }
67 file_checksum(fname,sum,st->st_size);
68 if (remote_version < 21) {
69 return (memcmp(sum,file->sum,2) == 0);
70 } else {
71 return (memcmp(sum,file->sum,MD4_SUM_LENGTH) == 0);
72 }
73 }
74
75 if (size_only) {
76 return 1;
77 }
78
79 if (ignore_times) {
80 return 0;
81 }
82
83 return (cmp_modtime(st->st_mtime,file->modtime) == 0);
84}
85
86
87/* use a larger block size for really big files */
88static int adapt_block_size(struct file_struct *file, int bsize)
89{
90 int ret;
91
92 if (bsize != BLOCK_SIZE) return bsize;
93
94 ret = file->length / (10000); /* rough heuristic */
95 ret = ret & ~15; /* multiple of 16 */
96 if (ret < bsize) ret = bsize;
97 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
98 return ret;
99}
100
101
102/*
103 send a sums struct down a fd
104 */
105static void send_sums(struct sum_struct *s, int f_out)
106{
107 if (s) {
108 size_t i;
109
110 /* tell the other guy how many we are going to be
111 doing and how many bytes there are in the last
112 chunk */
113 write_int(f_out, s->count);
114 write_int(f_out, s->n);
115 write_int(f_out, s->remainder);
116
117 for (i = 0; i < s->count; i++) {
118 write_int(f_out, s->sums[i].sum1);
119 write_buf(f_out, s->sums[i].sum2, csum_length);
120 }
121 } else {
122 /* we don't have checksums */
123 write_int(f_out, 0);
124 write_int(f_out, block_size);
125 write_int(f_out, 0);
126 }
127}
128
129
130/**
131 * Perhaps we want to just send an empty checksum set for this file,
132 * which will force the whole thing to be literally transferred.
133 *
134 * When do we do this? If the user's explicitly said they
135 * want the whole thing, or if { they haven't explicitly
136 * requested a delta, and it's local but not batch mode.}
137 *
138 * Whew. */
139static BOOL disable_deltas_p(void)
140{
141 extern int whole_file, no_whole_file;
142 extern int local_server;
143 extern int write_batch;
144
145 assert(whole_file == 0 || whole_file == 1);
146
147 /* whole_file and no_whole_file are never both on at the same time */
148
149 if (whole_file)
150 return True;
151 else if (no_whole_file)
152 return False;
153 else if (write_batch)
154 return False;
155 else
156 return local_server;
157}
158
159
160/*
161 generate a stream of signatures/checksums that describe a buffer
162
163 generate approximately one checksum every n bytes
164 */
165static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
166{
167 int i;
168 struct sum_struct *s;
169 int count;
170 int block_len = n;
171 int remainder = (len%block_len);
172 OFF_T offset = 0;
173
174 count = (len+(block_len-1))/block_len;
175
176 s = (struct sum_struct *)malloc(sizeof(*s));
177 if (!s) out_of_memory("generate_sums");
178
179 s->count = count;
180 s->remainder = remainder;
181 s->n = n;
182 s->flength = len;
183
184 if (count==0) {
185 s->sums = NULL;
186 return s;
187 }
188
189 if (verbose > 3)
190 rprintf(FINFO,"count=%d rem=%d n=%d flength=%.0f\n",
191 s->count,s->remainder,s->n,(double)s->flength);
192
193 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
194 if (!s->sums) out_of_memory("generate_sums");
195
196 for (i=0;i<count;i++) {
197 int n1 = MIN(len,n);
198 char *map = map_ptr(buf,offset,n1);
199
200 s->sums[i].sum1 = get_checksum1(map,n1);
201 get_checksum2(map,n1,s->sums[i].sum2);
202
203 s->sums[i].offset = offset;
204 s->sums[i].len = n1;
205 s->sums[i].i = i;
206
207 if (verbose > 3)
208 rprintf(FINFO,"chunk[%d] offset=%.0f len=%d sum1=%08x\n",
209 i,(double)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
210
211 len -= n1;
212 offset += n1;
213 }
214
215 return s;
216}
217
218
219
220/*
221 * Acts on file number I from FLIST, whose name is fname.
222 *
223 * First fixes up permissions, then generates checksums for the file.
224 *
225 * (This comment was added later by mbp who was trying to work it out;
226 * it might be wrong.)
227 */
228void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
229{
230 int fd;
231 STRUCT_STAT st;
232 struct map_struct *buf;
233 struct sum_struct *s;
234 int statret;
235 struct file_struct *file = flist->files[i];
236 char *fnamecmp;
237 char fnamecmpbuf[MAXPATHLEN];
238 extern char *compare_dest;
239 extern int list_only;
240 extern int preserve_perms;
241 extern int only_existing;
242
243 if (list_only) return;
244
245 if (verbose > 2)
246 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
247
248 statret = link_stat(fname,&st);
249
250 if (only_existing && statret == -1 && errno == ENOENT) {
251 /* we only want to update existing files */
252 if (verbose > 1) rprintf(FINFO, "not creating new file \"%s\"\n",fname);
253 return;
254 }
255
256 if (statret == 0 &&
257 !preserve_perms &&
258 (S_ISDIR(st.st_mode) == S_ISDIR(file->mode))) {
259 /* if the file exists already and we aren't perserving
260 presmissions then act as though the remote end sent
261 us the file permissions we already have */
262 file->mode = (file->mode & _S_IFMT) | (st.st_mode & ~_S_IFMT);
263 }
264
265 if (S_ISDIR(file->mode)) {
266 /* The file to be received is a directory, so we need
267 * to prepare appropriately. If there is already a
268 * file of that name and it is *not* a directory, then
269 * we need to delete it. If it doesn't exist, then
270 * recursively create it. */
271
272 if (dry_run) return; /* XXXX -- might cause inaccuracies?? -- mbp */
273 if (statret == 0 && !S_ISDIR(st.st_mode)) {
274 if (robust_unlink(fname) != 0) {
275 rprintf(FERROR, RSYNC_NAME
276 ": recv_generator: unlink \"%s\" to make room for directory: %s\n",
277 fname,strerror(errno));
278 return;
279 }
280 statret = -1;
281 }
282 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
283 if (!(relative_paths && errno==ENOENT &&
284 create_directory_path(fname)==0 &&
285 do_mkdir(fname,file->mode)==0)) {
286 rprintf(FERROR, RSYNC_NAME ": recv_generator: mkdir \"%s\": %s (2)\n",
287 fname,strerror(errno));
288 }
289 }
290 /* f_out is set to -1 when doing final directory
291 permission and modification time repair */
292 if (set_perms(fname,file,NULL,0) && verbose && (f_out != -1))
293 rprintf(FINFO,"%s/\n",fname);
294 return;
295 }
296
297 if (preserve_links && S_ISLNK(file->mode)) {
298#if SUPPORT_LINKS
299 char lnk[MAXPATHLEN];
300 int l;
301 extern int safe_symlinks;
302
303 if (safe_symlinks && unsafe_symlink(file->link, fname)) {
304 if (verbose) {
305 rprintf(FINFO,"ignoring unsafe symlink \"%s\" -> \"%s\"\n",
306 fname,file->link);
307 }
308 return;
309 }
310 if (statret == 0) {
311 l = readlink(fname,lnk,MAXPATHLEN-1);
312 if (l > 0) {
313 lnk[l] = 0;
314 /* A link already pointing to the
315 * right place -- no further action
316 * required. */
317 if (strcmp(lnk,file->link) == 0) {
318 set_perms(fname,file,&st,1);
319 return;
320 }
321 }
322 /* Not a symlink, so delete whatever's
323 * already there and put a new symlink
324 * in place. */
325 delete_file(fname);
326 }
327 if (do_symlink(file->link,fname) != 0) {
328 rprintf(FERROR,RSYNC_NAME": symlink \"%s\" -> \"%s\": %s\n",
329 fname,file->link,strerror(errno));
330 } else {
331 set_perms(fname,file,NULL,0);
332 if (verbose) {
333 rprintf(FINFO,"%s -> %s\n", fname,file->link);
334 }
335 }
336#endif
337 return;
338 }
339
340#ifdef HAVE_MKNOD
341 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
342 if (statret != 0 ||
343 st.st_mode != file->mode ||
344 st.st_rdev != file->rdev) {
345 delete_file(fname);
346 if (verbose > 2)
347 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
348 fname,(int)file->mode,(int)file->rdev);
349 if (do_mknod(fname,file->mode,file->rdev) != 0) {
350 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
351 } else {
352 set_perms(fname,file,NULL,0);
353 if (verbose)
354 rprintf(FINFO,"%s\n",fname);
355 }
356 } else {
357 set_perms(fname,file,&st,1);
358 }
359 return;
360 }
361#endif
362
363 if (preserve_hard_links && check_hard_link(file)) {
364 if (verbose > 1)
365 rprintf(FINFO, "\"%s\" is a hard link\n",f_name(file));
366 return;
367 }
368
369 if (!S_ISREG(file->mode)) {
370 rprintf(FINFO, "skipping non-regular file \"%s\"\n",fname);
371 return;
372 }
373
374 fnamecmp = fname;
375
376 if ((statret == -1) && (compare_dest != NULL)) {
377 /* try the file at compare_dest instead */
378 int saveerrno = errno;
379 snprintf(fnamecmpbuf,MAXPATHLEN,"%s/%s",compare_dest,fname);
380 statret = link_stat(fnamecmpbuf,&st);
381 if (!S_ISREG(st.st_mode))
382 statret = -1;
383 if (statret == -1)
384 errno = saveerrno;
385 else
386 fnamecmp = fnamecmpbuf;
387 }
388
389 if (statret == -1) {
390 if (errno == ENOENT) {
391 write_int(f_out,i);
392 if (!dry_run) send_sums(NULL,f_out);
393 } else {
394 if (verbose > 1)
395 rprintf(FERROR, RSYNC_NAME
396 ": recv_generator failed to open \"%s\": %s\n",
397 fname, strerror(errno));
398 }
399 return;
400 }
401
402 if (!S_ISREG(st.st_mode)) {
403 if (delete_file(fname) != 0) {
404 return;
405 }
406
407 /* now pretend the file didn't exist */
408 write_int(f_out,i);
409 if (!dry_run) send_sums(NULL,f_out);
410 return;
411 }
412
413 if (opt_ignore_existing && fnamecmp == fname) {
414 if (verbose > 1)
415 rprintf(FINFO,"%s exists\n",fname);
416 return;
417 }
418
419 if (update_only && cmp_modtime(st.st_mtime,file->modtime)>0 && fnamecmp == fname) {
420 if (verbose > 1)
421 rprintf(FINFO,"%s is newer\n",fname);
422 return;
423 }
424
425 if (skip_file(fname, file, &st)) {
426 if (fnamecmp == fname)
427 set_perms(fname,file,&st,1);
428 return;
429 }
430
431 if (dry_run) {
432 write_int(f_out,i);
433 return;
434 }
435
436 if (disable_deltas_p()) {
437 write_int(f_out,i);
438 send_sums(NULL,f_out);
439 return;
440 }
441
442 /* open the file */
443 fd = do_open(fnamecmp, O_RDONLY, 0);
444
445 if (fd == -1) {
446 rprintf(FERROR,RSYNC_NAME": failed to open \"%s\", continuing : %s\n",fnamecmp,strerror(errno));
447 /* pretend the file didn't exist */
448 write_int(f_out,i);
449 send_sums(NULL,f_out);
450 return;
451 }
452
453 if (st.st_size > 0) {
454 buf = map_file(fd,st.st_size);
455 } else {
456 buf = NULL;
457 }
458
459 if (verbose > 3)
460 rprintf(FINFO,"gen mapped %s of size %.0f\n",fnamecmp,(double)st.st_size);
461
462 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
463
464 if (verbose > 2)
465 rprintf(FINFO,"sending sums for %d\n",i);
466
467 write_int(f_out,i);
468 send_sums(s,f_out);
469
470 close(fd);
471 if (buf) unmap_file(buf);
472
473 free_sums(s);
474}
475
476
477
478void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
479{
480 int i;
481 int phase=0;
482
483 if (verbose > 2)
484 rprintf(FINFO,"generator starting pid=%d count=%d\n",
485 (int)getpid(),flist->count);
486
487 /* we expect to just sit around now, so don't exit on a
488 timeout. If we really get a timeout then the other process should
489 exit */
490 io_timeout = 0;
491
492 for (i = 0; i < flist->count; i++) {
493 struct file_struct *file = flist->files[i];
494 mode_t saved_mode = file->mode;
495 if (!file->basename) continue;
496
497 /* we need to ensure that any directories we create have writeable
498 permissions initially so that we can create the files within
499 them. This is then fixed after the files are transferred */
500 if (!am_root && S_ISDIR(file->mode)) {
501 file->mode |= S_IWUSR; /* user write */
502 /* XXX: Could this be causing a problem on SCO? Perhaps their
503 * handling of permissions is strange? */
504 }
505
506 recv_generator(local_name?local_name:f_name(file),
507 flist,i,f);
508
509 file->mode = saved_mode;
510 }
511
512 phase++;
513 csum_length = SUM_LENGTH;
514 ignore_times=1;
515
516 if (verbose > 2)
517 rprintf(FINFO,"generate_files phase=%d\n",phase);
518
519 write_int(f,-1);
520
521 if (remote_version >= 13) {
522 /* in newer versions of the protocol the files can cycle through
523 the system more than once to catch initial checksum errors */
524 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
525 struct file_struct *file = flist->files[i];
526 recv_generator(local_name?local_name:f_name(file),
527 flist,i,f);
528 }
529
530 phase++;
531 if (verbose > 2)
532 rprintf(FINFO,"generate_files phase=%d\n",phase);
533
534 write_int(f,-1);
535 }
536}