2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
56 static void free_sums(struct sum_struct *s)
58 if (s->sums) free(s->sums);
65 send a sums struct down a fd
67 static void send_sums(struct sum_struct *s,int f_out)
71 /* tell the other guy how many we are going to be doing and how many
72 bytes there are in the last chunk */
73 write_int(f_out,s?s->count:0);
74 write_int(f_out,s?s->n:block_size);
75 write_int(f_out,s?s->remainder:0);
77 for (i=0;i<s->count;i++) {
78 write_int(f_out,s->sums[i].sum1);
79 write_buf(f_out,s->sums[i].sum2,csum_length);
86 generate a stream of signatures/checksums that describe a buffer
88 generate approximately one checksum every n bytes
90 static struct sum_struct *generate_sums(struct map_struct *buf,off_t len,int n)
96 int remainder = (len%block_len);
99 count = (len+(block_len-1))/block_len;
101 s = (struct sum_struct *)malloc(sizeof(*s));
102 if (!s) out_of_memory("generate_sums");
105 s->remainder = remainder;
115 fprintf(FERROR,"count=%d rem=%d n=%d flength=%d\n",
116 s->count,s->remainder,s->n,(int)s->flength);
118 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
119 if (!s->sums) out_of_memory("generate_sums");
121 for (i=0;i<count;i++) {
123 char *map = map_ptr(buf,offset,n1);
125 s->sums[i].sum1 = get_checksum1(map,n1);
126 get_checksum2(map,n1,s->sums[i].sum2);
128 s->sums[i].offset = offset;
133 fprintf(FERROR,"chunk[%d] offset=%d len=%d sum1=%08x\n",
134 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
145 receive the checksums for a buffer
147 static struct sum_struct *receive_sums(int f)
149 struct sum_struct *s;
153 s = (struct sum_struct *)malloc(sizeof(*s));
154 if (!s) out_of_memory("receive_sums");
156 s->count = read_int(f);
158 s->remainder = read_int(f);
162 fprintf(FERROR,"count=%d n=%d rem=%d\n",
163 s->count,s->n,s->remainder);
168 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
169 if (!s->sums) out_of_memory("receive_sums");
171 for (i=0;i<s->count;i++) {
172 s->sums[i].sum1 = read_int(f);
173 read_buf(f,s->sums[i].sum2,csum_length);
175 s->sums[i].offset = offset;
178 if (i == s->count-1 && s->remainder != 0) {
179 s->sums[i].len = s->remainder;
181 s->sums[i].len = s->n;
183 offset += s->sums[i].len;
186 fprintf(FERROR,"chunk[%d] len=%d offset=%d sum1=%08x\n",
187 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
196 static int set_perms(char *fname,struct file_struct *file,struct stat *st,
202 if (dry_run) return 0;
205 if (link_stat(fname,&st2) != 0) {
206 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
212 if (preserve_times && !S_ISLNK(st->st_mode) &&
213 st->st_mtime != file->modtime) {
215 if (set_modtime(fname,file->modtime) != 0) {
216 fprintf(FERROR,"failed to set times on %s : %s\n",
217 fname,strerror(errno));
223 if (preserve_perms && !S_ISLNK(st->st_mode) &&
224 st->st_mode != file->mode) {
226 if (chmod(fname,file->mode) != 0) {
227 fprintf(FERROR,"failed to set permissions on %s : %s\n",
228 fname,strerror(errno));
234 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
235 (preserve_gid && st->st_gid != file->gid)) {
238 (am_root&&preserve_uid)?file->uid:-1,
239 preserve_gid?file->gid:-1) != 0) {
240 if (verbose>1 || preserve_uid)
241 fprintf(FERROR,"chown %s : %s\n",fname,strerror(errno));
246 if (verbose > 1 && report) {
248 fprintf(FINFO,"%s\n",fname);
250 fprintf(FINFO,"%s is uptodate\n",fname);
256 /* choose whether to skip a particular file */
257 static int skip_file(char *fname,
258 struct file_struct *file, struct stat *st)
260 if (st->st_size != file->length) {
264 /* if always checksum is set then we use the checksum instead
265 of the file time to determine whether to sync */
266 if (always_checksum && S_ISREG(st->st_mode)) {
267 char sum[MD4_SUM_LENGTH];
268 file_checksum(fname,sum,st->st_size);
269 return (memcmp(sum,file->sum,csum_length) == 0);
276 return (st->st_mtime == file->modtime);
280 /* use a larger block size for really big files */
281 int adapt_block_size(struct file_struct *file, int bsize)
283 int ret = file->length / (10000); /* rough heuristic */
284 ret = ret & ~15; /* multiple of 16 */
285 if (ret < bsize) ret = bsize;
286 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
290 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
294 struct map_struct *buf;
295 struct sum_struct *s;
297 struct file_struct *file = flist->files[i];
300 fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
302 statret = link_stat(fname,&st);
304 if (S_ISDIR(file->mode)) {
306 if (statret == 0 && !S_ISDIR(st.st_mode)) {
307 if (unlink(fname) != 0) {
308 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
313 if (statret != 0 && mkdir(fname,file->mode) != 0 && errno != EEXIST) {
314 if (!(relative_paths && errno==ENOENT &&
315 create_directory_path(fname)==0 &&
316 mkdir(fname,file->mode)==0)) {
317 fprintf(FERROR,"mkdir %s : %s (2)\n",
318 fname,strerror(errno));
321 if (set_perms(fname,file,NULL,0) && verbose)
322 fprintf(FINFO,"%s/\n",fname);
326 if (preserve_links && S_ISLNK(file->mode)) {
328 char lnk[MAXPATHLEN];
331 l = readlink(fname,lnk,MAXPATHLEN-1);
334 if (strcmp(lnk,file->link) == 0) {
335 set_perms(fname,file,&st,1);
340 if (!dry_run) unlink(fname);
341 if (!dry_run && symlink(file->link,fname) != 0) {
342 fprintf(FERROR,"link %s -> %s : %s\n",
343 fname,file->link,strerror(errno));
345 set_perms(fname,file,NULL,0);
347 fprintf(FINFO,"%s -> %s\n",
355 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
357 st.st_mode != file->mode ||
358 st.st_rdev != file->rdev) {
359 if (!dry_run) unlink(fname);
361 fprintf(FERROR,"mknod(%s,0%o,0x%x)\n",
362 fname,(int)file->mode,(int)file->rdev);
364 mknod(fname,file->mode,file->rdev) != 0) {
365 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
367 set_perms(fname,file,NULL,0);
369 fprintf(FINFO,"%s\n",fname);
372 set_perms(fname,file,&st,1);
378 if (preserve_hard_links && check_hard_link(file)) {
380 fprintf(FINFO,"%s is a hard link\n",f_name(file));
384 if (!S_ISREG(file->mode)) {
385 fprintf(FERROR,"skipping non-regular file %s\n",fname);
390 if (errno == ENOENT) {
392 if (!dry_run) send_sums(NULL,f_out);
395 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
400 if (!S_ISREG(st.st_mode)) {
401 /* its not a regular file on the receiving end, but it is on the
402 sending end. If its a directory then skip it (too dangerous to
403 do a recursive deletion??) otherwise try to unlink it */
404 if (S_ISDIR(st.st_mode)) {
405 fprintf(FERROR,"ERROR: %s is a directory\n",fname);
408 if (unlink(fname) != 0) {
409 fprintf(FERROR,"%s : not a regular file (generator)\n",fname);
413 /* now pretend the file didn't exist */
415 if (!dry_run) send_sums(NULL,f_out);
419 if (update_only && st.st_mtime > file->modtime) {
421 fprintf(FERROR,"%s is newer\n",fname);
425 if (skip_file(fname, file, &st)) {
426 set_perms(fname,file,&st,1);
437 send_sums(NULL,f_out);
442 fd = open(fname,O_RDONLY);
445 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
446 fprintf(FERROR,"skipping %s\n",fname);
450 if (st.st_size > 0) {
451 buf = map_file(fd,st.st_size);
457 fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
459 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
462 fprintf(FERROR,"sending sums for %d\n",i);
469 if (buf) unmap_file(buf);
476 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
478 int i,n,remainder,len,count;
482 static char file_sum1[MD4_SUM_LENGTH];
483 static char file_sum2[MD4_SUM_LENGTH];
486 count = read_int(f_in);
488 remainder = read_int(f_in);
492 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
495 fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
499 if (fd != -1 && write_sparse(fd,data,i) != i) {
500 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
508 if (i == count-1 && remainder != 0)
512 fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
513 i,len,(int)offset2,(int)offset);
515 map = map_ptr(buf,offset2,len);
520 if (fd != -1 && write_sparse(fd,map,len) != len) {
521 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
528 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
529 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
535 if (remote_version >= 14) {
536 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
538 fprintf(FERROR,"got file_sum\n");
539 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
546 static void delete_one(struct file_struct *f)
548 if (!S_ISDIR(f->mode)) {
549 if (!dry_run && unlink(f_name(f)) != 0) {
550 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
551 } else if (verbose) {
552 fprintf(FERROR,"deleting %s\n",f_name(f));
555 if (!dry_run && rmdir(f_name(f)) != 0) {
556 if (errno != ENOTEMPTY)
557 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
558 } else if (verbose) {
559 fprintf(FERROR,"deleting directory %s\n",f_name(f));
565 /* yuck! This function wouldn't have been necessary if I had the sorting
566 algorithm right. Unfortunately fixing the sorting algorithm would introduce
567 a backward incompatibility as file list indexes are sent over the link.
569 The aim is to see if a directory has already had the deletion algorithm applied
570 to it (due to recursion), and if so to skip it. The bisection is to
571 prevent this being an n^2 algorithm */
572 static int delete_already_done(struct file_list *flist,int j)
578 if (j == 0) return 0;
580 name = strdup(f_name(flist->files[j]));
583 fprintf(FERROR,"out of memory in delete_already_done");
587 p = strrchr(name,'/');
594 while (low != high) {
595 int mid = (low+high)/2;
596 int ret = strcmp(f_name(flist->files[flist_up(flist, mid)]),name);
608 low = flist_up(flist, low);
610 if (strcmp(f_name(flist->files[low]),name) == 0) {
620 /* this deletes any files on the receiving side that are not present
621 on the sending side. For version 1.6.4 I have changed the behaviour
622 to match more closely what most people seem to expect of this option */
623 static void delete_files(struct file_list *flist)
625 struct file_list *local_file_list;
631 for (j=0;j<flist->count;j++) {
632 char *name = f_name(flist->files[j]);
634 if (!S_ISDIR(flist->files[j]->mode)) continue;
636 if (delete_already_done(flist, j)) continue;
638 if (!(local_file_list = send_file_list(-1,1,&name)))
642 fprintf(FINFO,"deleting in %s\n", name);
644 for (i=local_file_list->count-1;i>=0;i--) {
645 if (!local_file_list->files[i]->basename) continue;
646 if (-1 == flist_find(flist,local_file_list->files[i])) {
647 delete_one(local_file_list->files[i]);
650 flist_free(local_file_list);
654 static char *cleanup_fname = NULL;
656 void exit_cleanup(int code)
659 unlink(cleanup_fname);
660 signal(SIGUSR1, SIG_IGN);
673 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
678 char fnametmp[MAXPATHLEN];
679 struct map_struct *buf;
681 struct file_struct *file;
686 fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
689 if (recurse && delete_mode && !local_name && flist->count>0) {
697 if (phase==0 && remote_version >= 13) {
699 csum_length = SUM_LENGTH;
701 fprintf(FERROR,"recv_files phase=%d\n",phase);
709 file = flist->files[i];
710 fname = f_name(file);
716 if (!am_server && verbose)
717 printf("%s\n",fname);
722 fprintf(FERROR,"recv_files(%s)\n",fname);
725 fd1 = open(fname,O_RDONLY);
727 if (fd1 != -1 && fstat(fd1,&st) != 0) {
728 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
729 receive_data(f_in,NULL,-1,NULL);
734 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
735 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
736 receive_data(f_in,NULL,-1,NULL);
741 if (fd1 != -1 && st.st_size > 0) {
742 buf = map_file(fd1,st.st_size);
744 fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
750 if (strlen(fname) > (MAXPATHLEN-8)) {
751 fprintf(FERROR,"filename too long\n");
752 if (buf) unmap_file(buf);
758 f = strrchr(fname,'/');
763 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
765 sprintf(fnametmp,"%s.XXXXXX",fname);
767 if (NULL == mktemp(fnametmp)) {
768 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
769 receive_data(f_in,buf,-1,NULL);
770 if (buf) unmap_file(buf);
774 fd2 = open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
775 if (fd2 == -1 && relative_paths && errno == ENOENT &&
776 create_directory_path(fnametmp) == 0) {
777 fd2 = open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
780 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
781 receive_data(f_in,buf,-1,NULL);
782 if (buf) unmap_file(buf);
787 cleanup_fname = fnametmp;
789 if (!am_server && verbose)
790 printf("%s\n",fname);
793 recv_ok = receive_data(f_in,buf,fd2,fname);
795 if (buf) unmap_file(buf);
802 fprintf(FERROR,"renaming %s to %s\n",fnametmp,fname);
805 char fnamebak[MAXPATHLEN];
806 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
807 fprintf(FERROR,"backup filename too long\n");
810 sprintf(fnamebak,"%s%s",fname,backup_suffix);
811 if (rename(fname,fnamebak) != 0 && errno != ENOENT) {
812 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
817 /* move tmp file over real file */
818 if (rename(fnametmp,fname) != 0) {
819 if (errno == EXDEV) {
820 /* rename failed on cross-filesystem link.
821 Copy the file instead. */
822 if (copy_file(fnametmp,fname, file->mode)) {
823 fprintf(FERROR,"copy %s -> %s : %s\n",
824 fnametmp,fname,strerror(errno));
826 set_perms(fname,file,NULL,0);
830 fprintf(FERROR,"rename %s -> %s : %s\n",
831 fnametmp,fname,strerror(errno));
835 set_perms(fname,file,NULL,0);
838 cleanup_fname = NULL;
842 if (csum_length == SUM_LENGTH) {
843 fprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
847 fprintf(FERROR,"redoing %s(%d)\n",fname,i);
853 if (preserve_hard_links)
854 do_hard_links(flist);
856 /* now we need to fix any directory permissions that were
857 modified during the transfer */
858 for (i = 0; i < flist->count; i++) {
859 struct file_struct *file = flist->files[i];
860 if (!file->basename || !S_ISDIR(file->mode)) continue;
861 recv_generator(f_name(file),flist,i,-1);
865 fprintf(FERROR,"recv_files finished\n");
872 off_t send_files(struct file_list *flist,int f_out,int f_in)
875 struct sum_struct *s;
876 struct map_struct *buf;
878 char fname[MAXPATHLEN];
881 struct file_struct *file;
885 fprintf(FERROR,"send_files starting\n");
887 setup_nonblocking(f_in,f_out);
892 if (phase==0 && remote_version >= 13) {
894 csum_length = SUM_LENGTH;
898 fprintf(FERROR,"send_files phase=%d\n",phase);
904 file = flist->files[i];
908 strncpy(fname,file->basedir,MAXPATHLEN-1);
909 fname[MAXPATHLEN-1] = 0;
910 if (strlen(fname) == MAXPATHLEN-1) {
911 fprintf(FERROR, "send_files failed on long-named directory %s\n",
917 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
920 fprintf(FERROR,"send_files(%d,%s)\n",i,fname);
923 if (!am_server && verbose)
924 printf("%s\n",fname);
929 s = receive_sums(f_in);
931 fprintf(FERROR,"receive_sums failed\n");
935 fd = open(fname,O_RDONLY);
937 fprintf(FERROR,"send_files failed to open %s: %s\n",
938 fname,strerror(errno));
942 /* map the local file */
943 if (fstat(fd,&st) != 0) {
944 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
949 if (st.st_size > 0) {
950 buf = map_file(fd,st.st_size);
956 fprintf(FERROR,"send_files mapped %s of size %d\n",
957 fname,(int)st.st_size);
961 write_int(f_out,s->count);
962 write_int(f_out,s->n);
963 write_int(f_out,s->remainder);
966 fprintf(FERROR,"calling match_sums %s\n",fname);
968 if (!am_server && verbose)
969 printf("%s\n",fname);
971 match_sums(f_out,s,buf,st.st_size);
974 if (buf) unmap_file(buf);
980 fprintf(FERROR,"sender finished %s\n",fname);
986 fprintf(FERROR,"send files finished\n");
998 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1004 fprintf(FERROR,"generator starting pid=%d count=%d\n",
1005 (int)getpid(),flist->count);
1007 for (i = 0; i < flist->count; i++) {
1008 struct file_struct *file = flist->files[i];
1009 mode_t saved_mode = file->mode;
1010 if (!file->basename) continue;
1012 /* we need to ensure that any directories we create have writeable
1013 permissions initially so that we can create the files within
1014 them. This is then fixed after the files are transferred */
1015 if (!am_root && S_ISDIR(file->mode)) {
1016 file->mode |= S_IWUSR; /* user write */
1019 recv_generator(local_name?local_name:f_name(file),
1022 file->mode = saved_mode;
1026 csum_length = SUM_LENGTH;
1030 fprintf(FERROR,"generate_files phase=%d\n",phase);
1035 if (remote_version >= 13) {
1036 /* in newer versions of the protocol the files can cycle through
1037 the system more than once to catch initial checksum errors */
1038 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1039 struct file_struct *file = flist->files[i];
1040 recv_generator(local_name?local_name:f_name(file),
1046 fprintf(FERROR,"generate_files phase=%d\n",phase);
1054 fprintf(FERROR,"generator wrote %d\n",write_total());