2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
56 static void free_sums(struct sum_struct *s)
58 if (s->sums) free(s->sums);
65 send a sums struct down a fd
67 static void send_sums(struct sum_struct *s,int f_out)
71 /* tell the other guy how many we are going to be doing and how many
72 bytes there are in the last chunk */
73 write_int(f_out,s?s->count:0);
74 write_int(f_out,s?s->n:block_size);
75 write_int(f_out,s?s->remainder:0);
77 for (i=0;i<s->count;i++) {
78 write_int(f_out,s->sums[i].sum1);
79 write_buf(f_out,s->sums[i].sum2,csum_length);
86 generate a stream of signatures/checksums that describe a buffer
88 generate approximately one checksum every n bytes
90 static struct sum_struct *generate_sums(struct map_struct *buf,off_t len,int n)
96 int remainder = (len%block_len);
99 count = (len+(block_len-1))/block_len;
101 s = (struct sum_struct *)malloc(sizeof(*s));
102 if (!s) out_of_memory("generate_sums");
105 s->remainder = remainder;
115 fprintf(FERROR,"count=%d rem=%d n=%d flength=%d\n",
116 s->count,s->remainder,s->n,(int)s->flength);
118 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
119 if (!s->sums) out_of_memory("generate_sums");
121 for (i=0;i<count;i++) {
123 char *map = map_ptr(buf,offset,n1);
125 s->sums[i].sum1 = get_checksum1(map,n1);
126 get_checksum2(map,n1,s->sums[i].sum2);
128 s->sums[i].offset = offset;
133 fprintf(FERROR,"chunk[%d] offset=%d len=%d sum1=%08x\n",
134 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
145 receive the checksums for a buffer
147 static struct sum_struct *receive_sums(int f)
149 struct sum_struct *s;
153 s = (struct sum_struct *)malloc(sizeof(*s));
154 if (!s) out_of_memory("receive_sums");
156 s->count = read_int(f);
158 s->remainder = read_int(f);
162 fprintf(FERROR,"count=%d n=%d rem=%d\n",
163 s->count,s->n,s->remainder);
168 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
169 if (!s->sums) out_of_memory("receive_sums");
171 for (i=0;i<s->count;i++) {
172 s->sums[i].sum1 = read_int(f);
173 read_buf(f,s->sums[i].sum2,csum_length);
175 s->sums[i].offset = offset;
178 if (i == s->count-1 && s->remainder != 0) {
179 s->sums[i].len = s->remainder;
181 s->sums[i].len = s->n;
183 offset += s->sums[i].len;
186 fprintf(FERROR,"chunk[%d] len=%d offset=%d sum1=%08x\n",
187 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
196 static int set_perms(char *fname,struct file_struct *file,struct stat *st,
202 if (dry_run) return 0;
205 if (link_stat(fname,&st2) != 0) {
206 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
212 if (preserve_times && !S_ISLNK(st->st_mode) &&
213 st->st_mtime != file->modtime) {
215 if (set_modtime(fname,file->modtime) != 0) {
216 fprintf(FERROR,"failed to set times on %s : %s\n",
217 fname,strerror(errno));
223 if (preserve_perms && !S_ISLNK(st->st_mode) &&
224 st->st_mode != file->mode) {
226 if (chmod(fname,file->mode) != 0) {
227 fprintf(FERROR,"failed to set permissions on %s : %s\n",
228 fname,strerror(errno));
234 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
235 (preserve_gid && st->st_gid != file->gid)) {
238 (am_root&&preserve_uid)?file->uid:-1,
239 preserve_gid?file->gid:-1) != 0) {
240 if (verbose>1 || preserve_uid)
241 fprintf(FERROR,"chown %s : %s\n",fname,strerror(errno));
246 if (verbose > 1 && report) {
248 fprintf(FINFO,"%s\n",fname);
250 fprintf(FINFO,"%s is uptodate\n",fname);
256 /* choose whether to skip a particular file */
257 static int skip_file(char *fname,
258 struct file_struct *file, struct stat *st)
260 if (st->st_size != file->length) {
264 /* if always checksum is set then we use the checksum instead
265 of the file time to determine whether to sync */
266 if (always_checksum && S_ISREG(st->st_mode)) {
267 char sum[MD4_SUM_LENGTH];
268 file_checksum(fname,sum,st->st_size);
269 return (memcmp(sum,file->sum,csum_length) == 0);
276 return (st->st_mtime == file->modtime);
280 /* use a larger block size for really big files */
281 int adapt_block_size(struct file_struct *file, int bsize)
283 int ret = file->length / (10000); /* rough heuristic */
284 ret = ret & ~15; /* multiple of 16 */
285 if (ret < bsize) ret = bsize;
289 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
293 struct map_struct *buf;
294 struct sum_struct *s;
296 struct file_struct *file = flist->files[i];
299 fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
301 statret = link_stat(fname,&st);
303 if (S_ISDIR(file->mode)) {
305 if (statret == 0 && !S_ISDIR(st.st_mode)) {
306 if (unlink(fname) != 0) {
307 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
312 if (statret != 0 && mkdir(fname,file->mode) != 0 && errno != EEXIST) {
313 if (!(relative_paths && errno==ENOENT &&
314 create_directory_path(fname)==0 &&
315 mkdir(fname,file->mode)==0)) {
316 fprintf(FERROR,"mkdir %s : %s (2)\n",
317 fname,strerror(errno));
320 if (set_perms(fname,file,NULL,0) && verbose)
321 fprintf(FINFO,"%s/\n",fname);
325 if (preserve_links && S_ISLNK(file->mode)) {
327 char lnk[MAXPATHLEN];
330 l = readlink(fname,lnk,MAXPATHLEN-1);
333 if (strcmp(lnk,file->link) == 0) {
334 set_perms(fname,file,&st,1);
339 if (!dry_run) unlink(fname);
340 if (!dry_run && symlink(file->link,fname) != 0) {
341 fprintf(FERROR,"link %s -> %s : %s\n",
342 fname,file->link,strerror(errno));
344 set_perms(fname,file,NULL,0);
346 fprintf(FINFO,"%s -> %s\n",
354 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
356 st.st_mode != file->mode ||
357 st.st_rdev != file->rdev) {
358 if (!dry_run) unlink(fname);
360 fprintf(FERROR,"mknod(%s,0%o,0x%x)\n",
361 fname,(int)file->mode,(int)file->rdev);
363 mknod(fname,file->mode,file->rdev) != 0) {
364 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
366 set_perms(fname,file,NULL,0);
368 fprintf(FINFO,"%s\n",fname);
371 set_perms(fname,file,&st,1);
377 if (preserve_hard_links && check_hard_link(file)) {
379 fprintf(FINFO,"%s is a hard link\n",f_name(file));
383 if (!S_ISREG(file->mode)) {
384 fprintf(FERROR,"skipping non-regular file %s\n",fname);
389 if (errno == ENOENT) {
391 if (!dry_run) send_sums(NULL,f_out);
394 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
399 if (!S_ISREG(st.st_mode)) {
400 /* its not a regular file on the receiving end, but it is on the
401 sending end. If its a directory then skip it (too dangerous to
402 do a recursive deletion??) otherwise try to unlink it */
403 if (S_ISDIR(st.st_mode)) {
404 fprintf(FERROR,"ERROR: %s is a directory\n",fname);
407 if (unlink(fname) != 0) {
408 fprintf(FERROR,"%s : not a regular file (generator)\n",fname);
412 /* now pretend the file didn't exist */
414 if (!dry_run) send_sums(NULL,f_out);
418 if (update_only && st.st_mtime > file->modtime) {
420 fprintf(FERROR,"%s is newer\n",fname);
424 if (skip_file(fname, file, &st)) {
425 set_perms(fname,file,&st,1);
436 send_sums(NULL,f_out);
441 fd = open(fname,O_RDONLY);
444 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
445 fprintf(FERROR,"skipping %s\n",fname);
449 if (st.st_size > 0) {
450 buf = map_file(fd,st.st_size);
456 fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
458 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
461 fprintf(FERROR,"sending sums for %d\n",i);
468 if (buf) unmap_file(buf);
475 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
477 int i,n,remainder,len,count;
481 static char file_sum1[MD4_SUM_LENGTH];
482 static char file_sum2[MD4_SUM_LENGTH];
485 count = read_int(f_in);
487 remainder = read_int(f_in);
491 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
494 fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
498 if (fd != -1 && write_sparse(fd,data,i) != i) {
499 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
507 if (i == count-1 && remainder != 0)
511 fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
512 i,len,(int)offset2,(int)offset);
514 map = map_ptr(buf,offset2,len);
519 if (fd != -1 && write_sparse(fd,map,len) != len) {
520 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
527 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
528 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
534 if (remote_version >= 14) {
535 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
537 fprintf(FERROR,"got file_sum\n");
538 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
545 static void delete_one(struct file_struct *f)
547 if (!S_ISDIR(f->mode)) {
548 if (!dry_run && unlink(f_name(f)) != 0) {
549 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
550 } else if (verbose) {
551 fprintf(FERROR,"deleting %s\n",f_name(f));
554 if (!dry_run && rmdir(f_name(f)) != 0) {
555 if (errno != ENOTEMPTY)
556 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
557 } else if (verbose) {
558 fprintf(FERROR,"deleting directory %s\n",f_name(f));
564 /* yuck! This function wouldn't have been necessary if I had the sorting
565 algorithm right. Unfortunately fixing the sorting algorithm would introduce
566 a backward incompatibility as file list indexes are sent over the link.
568 The aim is to see if a directory has already had the deletion algorithm applied
569 to it (due to recursion), and if so to skip it. The bisection is to
570 prevent this being an n^2 algorithm */
571 static int delete_already_done(struct file_list *flist,int j)
577 if (j == 0) return 0;
579 name = strdup(f_name(flist->files[j]));
582 fprintf(FERROR,"out of memory in delete_already_done");
586 p = strrchr(name,'/');
593 while (low != high) {
594 int mid = (low+high)/2;
595 int ret = strcmp(f_name(flist->files[flist_up(flist, mid)]),name);
607 low = flist_up(flist, low);
609 if (strcmp(f_name(flist->files[low]),name) == 0) {
619 /* this deletes any files on the receiving side that are not present
620 on the sending side. For version 1.6.4 I have changed the behaviour
621 to match more closely what most people seem to expect of this option */
622 static void delete_files(struct file_list *flist)
624 struct file_list *local_file_list;
630 for (j=0;j<flist->count;j++) {
631 char *name = f_name(flist->files[j]);
633 if (!S_ISDIR(flist->files[j]->mode)) continue;
635 if (delete_already_done(flist, j)) continue;
637 if (!(local_file_list = send_file_list(-1,1,&name)))
641 fprintf(FINFO,"deleting in %s\n", name);
643 for (i=local_file_list->count-1;i>=0;i--) {
644 if (!local_file_list->files[i]->basename) continue;
645 if (-1 == flist_find(flist,local_file_list->files[i])) {
646 delete_one(local_file_list->files[i]);
649 flist_free(local_file_list);
653 static char *cleanup_fname = NULL;
655 void exit_cleanup(int code)
658 unlink(cleanup_fname);
659 signal(SIGUSR1, SIG_IGN);
672 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
677 char fnametmp[MAXPATHLEN];
678 struct map_struct *buf;
680 struct file_struct *file;
685 fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
688 if (recurse && delete_mode && !local_name && flist->count>0) {
696 if (phase==0 && remote_version >= 13) {
698 csum_length = SUM_LENGTH;
700 fprintf(FERROR,"recv_files phase=%d\n",phase);
708 file = flist->files[i];
709 fname = f_name(file);
715 if (!am_server && verbose)
716 printf("%s\n",fname);
721 fprintf(FERROR,"recv_files(%s)\n",fname);
724 fd1 = open(fname,O_RDONLY);
726 if (fd1 != -1 && fstat(fd1,&st) != 0) {
727 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
728 receive_data(f_in,NULL,-1,NULL);
733 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
734 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
735 receive_data(f_in,NULL,-1,NULL);
740 if (fd1 != -1 && st.st_size > 0) {
741 buf = map_file(fd1,st.st_size);
743 fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
749 if (strlen(fname) > (MAXPATHLEN-8)) {
750 fprintf(FERROR,"filename too long\n");
751 if (buf) unmap_file(buf);
757 f = strrchr(fname,'/');
762 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
764 sprintf(fnametmp,"%s.XXXXXX",fname);
766 if (NULL == mktemp(fnametmp)) {
767 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
768 receive_data(f_in,buf,-1,NULL);
769 if (buf) unmap_file(buf);
773 fd2 = open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
774 if (fd2 == -1 && relative_paths && errno == ENOENT &&
775 create_directory_path(fnametmp) == 0) {
776 fd2 = open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
779 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
780 receive_data(f_in,buf,-1,NULL);
781 if (buf) unmap_file(buf);
786 cleanup_fname = fnametmp;
788 if (!am_server && verbose)
789 printf("%s\n",fname);
792 recv_ok = receive_data(f_in,buf,fd2,fname);
794 if (buf) unmap_file(buf);
801 fprintf(FERROR,"renaming %s to %s\n",fnametmp,fname);
804 char fnamebak[MAXPATHLEN];
805 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
806 fprintf(FERROR,"backup filename too long\n");
809 sprintf(fnamebak,"%s%s",fname,backup_suffix);
810 if (rename(fname,fnamebak) != 0 && errno != ENOENT) {
811 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
816 /* move tmp file over real file */
817 if (rename(fnametmp,fname) != 0) {
818 if (errno == EXDEV) {
819 /* rename failed on cross-filesystem link.
820 Copy the file instead. */
821 if (copy_file(fnametmp,fname, file->mode)) {
822 fprintf(FERROR,"copy %s -> %s : %s\n",
823 fnametmp,fname,strerror(errno));
825 set_perms(fname,file,NULL,0);
829 fprintf(FERROR,"rename %s -> %s : %s\n",
830 fnametmp,fname,strerror(errno));
834 set_perms(fname,file,NULL,0);
837 cleanup_fname = NULL;
842 fprintf(FERROR,"redoing %s(%d)\n",fname,i);
843 if (csum_length == SUM_LENGTH)
844 fprintf(FERROR,"ERROR: file corruption in %s\n",fname);
849 if (preserve_hard_links)
850 do_hard_links(flist);
852 /* now we need to fix any directory permissions that were
853 modified during the transfer */
854 for (i = 0; i < flist->count; i++) {
855 struct file_struct *file = flist->files[i];
856 if (!file->basename || !S_ISDIR(file->mode)) continue;
857 recv_generator(f_name(file),flist,i,-1);
861 fprintf(FERROR,"recv_files finished\n");
868 off_t send_files(struct file_list *flist,int f_out,int f_in)
871 struct sum_struct *s;
872 struct map_struct *buf;
874 char fname[MAXPATHLEN];
877 struct file_struct *file;
881 fprintf(FERROR,"send_files starting\n");
883 setup_nonblocking(f_in,f_out);
889 if (phase==0 && remote_version >= 13) {
891 csum_length = SUM_LENGTH;
895 fprintf(FERROR,"send_files phase=%d\n",phase);
901 file = flist->files[i];
905 strncpy(fname,file->basedir,MAXPATHLEN-1);
906 fname[MAXPATHLEN-1] = 0;
907 if (strlen(fname) == MAXPATHLEN-1) {
908 fprintf(FERROR, "send_files failed on long-named directory %s\n",
914 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
917 fprintf(FERROR,"send_files(%d,%s)\n",i,fname);
920 if (!am_server && verbose)
921 printf("%s\n",fname);
926 s = receive_sums(f_in);
928 fprintf(FERROR,"receive_sums failed\n");
932 fd = open(fname,O_RDONLY);
934 fprintf(FERROR,"send_files failed to open %s: %s\n",
935 fname,strerror(errno));
939 /* map the local file */
940 if (fstat(fd,&st) != 0) {
941 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
946 if (st.st_size > 0) {
947 buf = map_file(fd,st.st_size);
953 fprintf(FERROR,"send_files mapped %s of size %d\n",
954 fname,(int)st.st_size);
958 write_int(f_out,s->count);
959 write_int(f_out,s->n);
960 write_int(f_out,s->remainder);
963 fprintf(FERROR,"calling match_sums %s\n",fname);
965 if (!am_server && verbose)
966 printf("%s\n",fname);
968 match_sums(f_out,s,buf,st.st_size);
971 if (buf) unmap_file(buf);
977 fprintf(FERROR,"sender finished %s\n",fname);
983 fprintf(FERROR,"send files finished\n");
995 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1001 fprintf(FERROR,"generator starting pid=%d count=%d\n",
1002 (int)getpid(),flist->count);
1004 for (i = 0; i < flist->count; i++) {
1005 struct file_struct *file = flist->files[i];
1006 mode_t saved_mode = file->mode;
1007 if (!file->basename) continue;
1009 /* we need to ensure that any directories we create have writeable
1010 permissions initially so that we can create the files within
1011 them. This is then fixed after the files are transferred */
1012 if (!am_root && S_ISDIR(file->mode)) {
1013 file->mode |= S_IWUSR; /* user write */
1016 recv_generator(local_name?local_name:f_name(file),
1019 file->mode = saved_mode;
1023 csum_length = SUM_LENGTH;
1027 fprintf(FERROR,"generate_files phase=%d\n",phase);
1032 if (remote_version >= 13) {
1033 /* in newer versions of the protocol the files can cycle through
1034 the system more than once to catch initial checksum errors */
1035 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1036 struct file_struct *file = flist->files[i];
1037 recv_generator(local_name?local_name:f_name(file),
1043 fprintf(FERROR,"generate_files phase=%d\n",phase);
1051 fprintf(FERROR,"generator wrote %d\n",write_total());