2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 rprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 rprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 rprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 slprintf(buf, sizeof(buf)-1, "%s/%s", fname, dname);
116 rprintf(FINFO,"deleting %s\n", buf);
117 if (delete_file(buf) != 0) {
125 if (do_rmdir(fname) != 0) {
126 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
134 send a sums struct down a fd
136 static void send_sums(struct sum_struct *s,int f_out)
140 /* tell the other guy how many we are going to be doing and how many
141 bytes there are in the last chunk */
142 write_int(f_out,s?s->count:0);
143 write_int(f_out,s?s->n:block_size);
144 write_int(f_out,s?s->remainder:0);
146 for (i=0;i<s->count;i++) {
147 write_int(f_out,s->sums[i].sum1);
148 write_buf(f_out,s->sums[i].sum2,csum_length);
155 generate a stream of signatures/checksums that describe a buffer
157 generate approximately one checksum every n bytes
159 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
162 struct sum_struct *s;
165 int remainder = (len%block_len);
168 count = (len+(block_len-1))/block_len;
170 s = (struct sum_struct *)malloc(sizeof(*s));
171 if (!s) out_of_memory("generate_sums");
174 s->remainder = remainder;
184 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
185 s->count,s->remainder,s->n,(int)s->flength);
187 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
188 if (!s->sums) out_of_memory("generate_sums");
190 for (i=0;i<count;i++) {
192 char *map = map_ptr(buf,offset,n1);
194 s->sums[i].sum1 = get_checksum1(map,n1);
195 get_checksum2(map,n1,s->sums[i].sum2);
197 s->sums[i].offset = offset;
202 rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
203 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
214 receive the checksums for a buffer
216 static struct sum_struct *receive_sums(int f)
218 struct sum_struct *s;
222 s = (struct sum_struct *)malloc(sizeof(*s));
223 if (!s) out_of_memory("receive_sums");
225 s->count = read_int(f);
227 s->remainder = read_int(f);
231 rprintf(FINFO,"count=%d n=%d rem=%d\n",
232 s->count,s->n,s->remainder);
237 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
238 if (!s->sums) out_of_memory("receive_sums");
240 for (i=0;i<s->count;i++) {
241 s->sums[i].sum1 = read_int(f);
242 read_buf(f,s->sums[i].sum2,csum_length);
244 s->sums[i].offset = offset;
247 if (i == s->count-1 && s->remainder != 0) {
248 s->sums[i].len = s->remainder;
250 s->sums[i].len = s->n;
252 offset += s->sums[i].len;
255 rprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
256 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
265 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
271 if (dry_run) return 0;
274 if (link_stat(fname,&st2) != 0) {
275 rprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
281 if (preserve_times && !S_ISLNK(st->st_mode) &&
282 st->st_mtime != file->modtime) {
284 if (set_modtime(fname,file->modtime) != 0) {
285 rprintf(FERROR,"failed to set times on %s : %s\n",
286 fname,strerror(errno));
292 if (preserve_perms && !S_ISLNK(st->st_mode) &&
293 st->st_mode != file->mode) {
295 if (do_chmod(fname,file->mode) != 0) {
296 rprintf(FERROR,"failed to set permissions on %s : %s\n",
297 fname,strerror(errno));
303 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
304 (preserve_gid && st->st_gid != file->gid)) {
306 (am_root&&preserve_uid)?file->uid:-1,
307 preserve_gid?file->gid:-1) != 0) {
308 if (preserve_uid && st->st_uid != file->uid)
310 if (verbose>1 || preserve_uid)
311 rprintf(FERROR,"chown %s : %s\n",
312 fname,strerror(errno));
318 if (verbose > 1 && report) {
320 rprintf(FINFO,"%s\n",fname);
322 rprintf(FINFO,"%s is uptodate\n",fname);
328 /* choose whether to skip a particular file */
329 static int skip_file(char *fname,
330 struct file_struct *file, STRUCT_STAT *st)
332 if (st->st_size != file->length) {
336 /* if always checksum is set then we use the checksum instead
337 of the file time to determine whether to sync */
338 if (always_checksum && S_ISREG(st->st_mode)) {
339 char sum[MD4_SUM_LENGTH];
340 file_checksum(fname,sum,st->st_size);
341 return (memcmp(sum,file->sum,csum_length) == 0);
348 return (st->st_mtime == file->modtime);
352 /* use a larger block size for really big files */
353 int adapt_block_size(struct file_struct *file, int bsize)
355 int ret = file->length / (10000); /* rough heuristic */
356 ret = ret & ~15; /* multiple of 16 */
357 if (ret < bsize) ret = bsize;
358 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
362 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
366 struct map_struct *buf;
367 struct sum_struct *s;
369 struct file_struct *file = flist->files[i];
372 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
374 statret = link_stat(fname,&st);
376 if (S_ISDIR(file->mode)) {
378 if (statret == 0 && !S_ISDIR(st.st_mode)) {
379 if (do_unlink(fname) != 0) {
380 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
385 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
386 if (!(relative_paths && errno==ENOENT &&
387 create_directory_path(fname)==0 &&
388 do_mkdir(fname,file->mode)==0)) {
389 rprintf(FERROR,"mkdir %s : %s (2)\n",
390 fname,strerror(errno));
393 if (set_perms(fname,file,NULL,0) && verbose)
394 rprintf(FINFO,"%s/\n",fname);
398 if (preserve_links && S_ISLNK(file->mode)) {
400 char lnk[MAXPATHLEN];
403 l = readlink(fname,lnk,MAXPATHLEN-1);
406 if (strcmp(lnk,file->link) == 0) {
407 set_perms(fname,file,&st,1);
413 if (do_symlink(file->link,fname) != 0) {
414 rprintf(FERROR,"link %s -> %s : %s\n",
415 fname,file->link,strerror(errno));
417 set_perms(fname,file,NULL,0);
419 rprintf(FINFO,"%s -> %s\n",
427 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
429 st.st_mode != file->mode ||
430 st.st_rdev != file->rdev) {
433 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
434 fname,(int)file->mode,(int)file->rdev);
435 if (do_mknod(fname,file->mode,file->rdev) != 0) {
436 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
438 set_perms(fname,file,NULL,0);
440 rprintf(FINFO,"%s\n",fname);
443 set_perms(fname,file,&st,1);
449 if (preserve_hard_links && check_hard_link(file)) {
451 rprintf(FINFO,"%s is a hard link\n",f_name(file));
455 if (!S_ISREG(file->mode)) {
456 rprintf(FINFO,"skipping non-regular file %s\n",fname);
461 if (errno == ENOENT) {
463 if (!dry_run) send_sums(NULL,f_out);
466 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
471 if (!S_ISREG(st.st_mode)) {
472 if (delete_file(fname) != 0) {
476 /* now pretend the file didn't exist */
478 if (!dry_run) send_sums(NULL,f_out);
482 if (update_only && st.st_mtime > file->modtime) {
484 rprintf(FINFO,"%s is newer\n",fname);
488 if (skip_file(fname, file, &st)) {
489 set_perms(fname,file,&st,1);
500 send_sums(NULL,f_out);
505 fd = open(fname,O_RDONLY);
508 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
509 rprintf(FERROR,"skipping %s\n",fname);
513 if (st.st_size > 0) {
514 buf = map_file(fd,st.st_size);
520 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
522 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
525 rprintf(FINFO,"sending sums for %d\n",i);
532 if (buf) unmap_file(buf);
539 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
541 int i,n,remainder,len,count;
545 static char file_sum1[MD4_SUM_LENGTH];
546 static char file_sum2[MD4_SUM_LENGTH];
549 count = read_int(f_in);
551 remainder = read_int(f_in);
555 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
558 rprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
562 if (fd != -1 && write_file(fd,data,i) != i) {
563 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
571 if (i == count-1 && remainder != 0)
575 rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
576 i,len,(int)offset2,(int)offset);
578 map = map_ptr(buf,offset2,len);
583 if (fd != -1 && write_file(fd,map,len) != len) {
584 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
591 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
592 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
598 if (remote_version >= 14) {
599 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
601 rprintf(FINFO,"got file_sum\n");
602 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
609 static void delete_one(struct file_struct *f)
611 if (!S_ISDIR(f->mode)) {
612 if (do_unlink(f_name(f)) != 0) {
613 rprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
614 } else if (verbose) {
615 rprintf(FINFO,"deleting %s\n",f_name(f));
618 if (do_rmdir(f_name(f)) != 0) {
619 if (errno != ENOTEMPTY && errno != EEXIST)
620 rprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
621 } else if (verbose) {
622 rprintf(FINFO,"deleting directory %s\n",f_name(f));
629 static struct delete_list {
633 static int dlist_len, dlist_alloc_len;
635 static void add_delete_entry(struct file_struct *file)
637 if (dlist_len == dlist_alloc_len) {
638 dlist_alloc_len += 1024;
640 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
642 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
644 if (!delete_list) out_of_memory("add_delete_entry");
647 delete_list[dlist_len].dev = file->dev;
648 delete_list[dlist_len].inode = file->inode;
652 rprintf(FINFO,"added %s to delete list\n", f_name(file));
655 /* yuck! This function wouldn't have been necessary if I had the sorting
656 algorithm right. Unfortunately fixing the sorting algorithm would introduce
657 a backward incompatibility as file list indexes are sent over the link.
659 static int delete_already_done(struct file_list *flist,int j)
664 if (link_stat(f_name(flist->files[j]), &st)) return 1;
666 for (i=0;i<dlist_len;i++) {
667 if (st.st_ino == delete_list[i].inode &&
668 st.st_dev == delete_list[i].dev)
676 /* this deletes any files on the receiving side that are not present
677 on the sending side. For version 1.6.4 I have changed the behaviour
678 to match more closely what most people seem to expect of this option */
679 static void delete_files(struct file_list *flist)
681 struct file_list *local_file_list;
689 rprintf(FINFO,"IO error encountered - skipping file deletion\n");
693 for (j=0;j<flist->count;j++) {
694 if (!S_ISDIR(flist->files[j]->mode) ||
695 !(flist->files[j]->flags & FLAG_DELETE)) continue;
697 if (delete_already_done(flist, j)) continue;
699 name = strdup(f_name(flist->files[j]));
701 if (!(local_file_list = send_file_list(-1,1,&name))) {
707 rprintf(FINFO,"deleting in %s\n", name);
709 for (i=local_file_list->count-1;i>=0;i--) {
710 if (!local_file_list->files[i]->basename) continue;
711 if (S_ISDIR(local_file_list->files[i]->mode))
712 add_delete_entry(local_file_list->files[i]);
713 if (-1 == flist_find(flist,local_file_list->files[i])) {
714 delete_one(local_file_list->files[i]);
717 flist_free(local_file_list);
722 static char *cleanup_fname;
724 void exit_cleanup(int code)
728 do_unlink(cleanup_fname);
729 signal(SIGUSR1, SIG_IGN);
742 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
747 char fnametmp[MAXPATHLEN];
748 struct map_struct *buf;
750 struct file_struct *file;
755 rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
758 if (recurse && delete_mode && !local_name && flist->count>0) {
766 if (phase==0 && remote_version >= 13) {
768 csum_length = SUM_LENGTH;
770 rprintf(FINFO,"recv_files phase=%d\n",phase);
778 file = flist->files[i];
779 fname = f_name(file);
785 if (!am_server && verbose)
786 printf("%s\n",fname);
791 rprintf(FINFO,"recv_files(%s)\n",fname);
794 fd1 = open(fname,O_RDONLY);
796 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
797 rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
798 receive_data(f_in,NULL,-1,NULL);
803 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
804 rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
805 receive_data(f_in,NULL,-1,NULL);
810 if (fd1 != -1 && st.st_size > 0) {
811 buf = map_file(fd1,st.st_size);
813 rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
821 f = strrchr(fname,'/');
826 if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) {
827 rprintf(FERROR,"filename too long\n");
828 if (buf) unmap_file(buf);
832 slprintf(fnametmp,sizeof(fnametmp)-1, "%s/.%s.XXXXXX",tmpdir,f);
834 char *f = strrchr(fname,'/');
836 if (strlen(fname)+9 > MAXPATHLEN) {
837 rprintf(FERROR,"filename too long\n");
838 if (buf) unmap_file(buf);
845 slprintf(fnametmp,sizeof(fnametmp)-1,"%s/.%s.XXXXXX",fname,f+1);
848 slprintf(fnametmp,sizeof(fnametmp)-1,".%s.XXXXXX",fname);
851 if (NULL == do_mktemp(fnametmp)) {
852 rprintf(FERROR,"mktemp %s failed\n",fnametmp);
853 receive_data(f_in,buf,-1,NULL);
854 if (buf) unmap_file(buf);
858 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
859 if (fd2 == -1 && relative_paths && errno == ENOENT &&
860 create_directory_path(fnametmp) == 0) {
861 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
864 rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
865 receive_data(f_in,buf,-1,NULL);
866 if (buf) unmap_file(buf);
871 cleanup_fname = fnametmp;
873 if (!am_server && verbose)
874 printf("%s\n",fname);
877 recv_ok = receive_data(f_in,buf,fd2,fname);
879 if (buf) unmap_file(buf);
886 rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
889 char fnamebak[MAXPATHLEN];
890 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
891 rprintf(FERROR,"backup filename too long\n");
894 slprintf(fnamebak,sizeof(fnamebak)-1,"%s%s",fname,backup_suffix);
895 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
896 rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
901 /* move tmp file over real file */
902 if (do_rename(fnametmp,fname) != 0) {
903 if (errno == EXDEV) {
904 /* rename failed on cross-filesystem link.
905 Copy the file instead. */
906 if (copy_file(fnametmp,fname, file->mode)) {
907 rprintf(FERROR,"copy %s -> %s : %s\n",
908 fnametmp,fname,strerror(errno));
910 set_perms(fname,file,NULL,0);
914 rprintf(FERROR,"rename %s -> %s : %s\n",
915 fnametmp,fname,strerror(errno));
919 set_perms(fname,file,NULL,0);
922 cleanup_fname = NULL;
926 if (csum_length == SUM_LENGTH) {
927 rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
931 rprintf(FINFO,"redoing %s(%d)\n",fname,i);
937 if (preserve_hard_links)
938 do_hard_links(flist);
940 /* now we need to fix any directory permissions that were
941 modified during the transfer */
942 for (i = 0; i < flist->count; i++) {
943 struct file_struct *file = flist->files[i];
944 if (!file->basename || !S_ISDIR(file->mode)) continue;
945 recv_generator(f_name(file),flist,i,-1);
949 rprintf(FINFO,"recv_files finished\n");
956 void send_files(struct file_list *flist,int f_out,int f_in)
959 struct sum_struct *s;
960 struct map_struct *buf;
962 char fname[MAXPATHLEN];
964 struct file_struct *file;
969 rprintf(FINFO,"send_files starting\n");
971 setup_nonblocking(f_in,f_out);
976 if (phase==0 && remote_version >= 13) {
978 csum_length = SUM_LENGTH;
982 rprintf(FINFO,"send_files phase=%d\n",phase);
988 file = flist->files[i];
992 strlcpy(fname,file->basedir,MAXPATHLEN-1);
993 if (strlen(fname) == MAXPATHLEN-1) {
995 rprintf(FERROR, "send_files failed on long-named directory %s\n",
999 strlcat(fname,"/",MAXPATHLEN-1);
1000 offset = strlen(file->basedir)+1;
1002 strlcat(fname,f_name(file),MAXPATHLEN-strlen(fname));
1005 rprintf(FINFO,"send_files(%d,%s)\n",i,fname);
1008 if (!am_server && verbose)
1009 printf("%s\n",fname);
1014 s = receive_sums(f_in);
1017 rprintf(FERROR,"receive_sums failed\n");
1021 fd = open(fname,O_RDONLY);
1024 rprintf(FERROR,"send_files failed to open %s: %s\n",
1025 fname,strerror(errno));
1030 /* map the local file */
1031 if (do_fstat(fd,&st) != 0) {
1033 rprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1039 if (st.st_size > 0) {
1040 buf = map_file(fd,st.st_size);
1046 rprintf(FINFO,"send_files mapped %s of size %d\n",
1047 fname,(int)st.st_size);
1051 write_int(f_out,s->count);
1052 write_int(f_out,s->n);
1053 write_int(f_out,s->remainder);
1056 rprintf(FINFO,"calling match_sums %s\n",fname);
1058 if (!am_server && verbose)
1059 printf("%s\n",fname+offset);
1061 match_sums(f_out,s,buf,st.st_size);
1064 if (buf) unmap_file(buf);
1070 rprintf(FINFO,"sender finished %s\n",fname);
1074 rprintf(FINFO,"send files finished\n");
1078 write_int(f_out,-1);
1084 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1090 rprintf(FINFO,"generator starting pid=%d count=%d\n",
1091 (int)getpid(),flist->count);
1093 for (i = 0; i < flist->count; i++) {
1094 struct file_struct *file = flist->files[i];
1095 mode_t saved_mode = file->mode;
1096 if (!file->basename) continue;
1098 /* we need to ensure that any directories we create have writeable
1099 permissions initially so that we can create the files within
1100 them. This is then fixed after the files are transferred */
1101 if (!am_root && S_ISDIR(file->mode)) {
1102 file->mode |= S_IWUSR; /* user write */
1105 recv_generator(local_name?local_name:f_name(file),
1108 file->mode = saved_mode;
1112 csum_length = SUM_LENGTH;
1116 rprintf(FINFO,"generate_files phase=%d\n",phase);
1121 /* we expect to just sit around now, so don't exit on a timeout. If we
1122 really get a timeout then the other process should exit */
1125 if (remote_version >= 13) {
1126 /* in newer versions of the protocol the files can cycle through
1127 the system more than once to catch initial checksum errors */
1128 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1129 struct file_struct *file = flist->files[i];
1130 recv_generator(local_name?local_name:f_name(file),
1136 rprintf(FINFO,"generate_files phase=%d\n",phase);
1144 rprintf(FINFO,"generator wrote %ld\n",(long)write_total());