2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 rprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 rprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 rprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 slprintf(buf, sizeof(buf)-1, "%s/%s", fname, dname);
116 rprintf(FINFO,"deleting %s\n", buf);
117 if (delete_file(buf) != 0) {
125 if (do_rmdir(fname) != 0) {
126 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
134 send a sums struct down a fd
136 static void send_sums(struct sum_struct *s,int f_out)
140 /* tell the other guy how many we are going to be doing and how many
141 bytes there are in the last chunk */
142 write_int(f_out,s?s->count:0);
143 write_int(f_out,s?s->n:block_size);
144 write_int(f_out,s?s->remainder:0);
146 for (i=0;i<s->count;i++) {
147 write_int(f_out,s->sums[i].sum1);
148 write_buf(f_out,s->sums[i].sum2,csum_length);
155 generate a stream of signatures/checksums that describe a buffer
157 generate approximately one checksum every n bytes
159 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
162 struct sum_struct *s;
165 int remainder = (len%block_len);
168 count = (len+(block_len-1))/block_len;
170 s = (struct sum_struct *)malloc(sizeof(*s));
171 if (!s) out_of_memory("generate_sums");
174 s->remainder = remainder;
184 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
185 s->count,s->remainder,s->n,(int)s->flength);
187 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
188 if (!s->sums) out_of_memory("generate_sums");
190 for (i=0;i<count;i++) {
192 char *map = map_ptr(buf,offset,n1);
194 s->sums[i].sum1 = get_checksum1(map,n1);
195 get_checksum2(map,n1,s->sums[i].sum2);
197 s->sums[i].offset = offset;
202 rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
203 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
214 receive the checksums for a buffer
216 static struct sum_struct *receive_sums(int f)
218 struct sum_struct *s;
222 s = (struct sum_struct *)malloc(sizeof(*s));
223 if (!s) out_of_memory("receive_sums");
225 s->count = read_int(f);
227 s->remainder = read_int(f);
231 rprintf(FINFO,"count=%d n=%d rem=%d\n",
232 s->count,s->n,s->remainder);
237 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
238 if (!s->sums) out_of_memory("receive_sums");
240 for (i=0;i<s->count;i++) {
241 s->sums[i].sum1 = read_int(f);
242 read_buf(f,s->sums[i].sum2,csum_length);
244 s->sums[i].offset = offset;
247 if (i == s->count-1 && s->remainder != 0) {
248 s->sums[i].len = s->remainder;
250 s->sums[i].len = s->n;
252 offset += s->sums[i].len;
255 rprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
256 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
265 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
271 if (dry_run) return 0;
274 if (link_stat(fname,&st2) != 0) {
275 rprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
281 if (preserve_times && !S_ISLNK(st->st_mode) &&
282 st->st_mtime != file->modtime) {
284 if (set_modtime(fname,file->modtime) != 0) {
285 rprintf(FERROR,"failed to set times on %s : %s\n",
286 fname,strerror(errno));
292 if (preserve_perms && !S_ISLNK(st->st_mode) &&
293 st->st_mode != file->mode) {
295 if (do_chmod(fname,file->mode) != 0) {
296 rprintf(FERROR,"failed to set permissions on %s : %s\n",
297 fname,strerror(errno));
303 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
304 (preserve_gid && st->st_gid != file->gid)) {
306 (am_root&&preserve_uid)?file->uid:-1,
307 preserve_gid?file->gid:-1) != 0) {
308 if (preserve_uid && st->st_uid != file->uid)
310 if (verbose>1 || preserve_uid)
311 rprintf(FERROR,"chown %s : %s\n",
312 fname,strerror(errno));
318 if (verbose > 1 && report) {
320 rprintf(FINFO,"%s\n",fname);
322 rprintf(FINFO,"%s is uptodate\n",fname);
328 /* choose whether to skip a particular file */
329 static int skip_file(char *fname,
330 struct file_struct *file, STRUCT_STAT *st)
332 if (st->st_size != file->length) {
336 /* if always checksum is set then we use the checksum instead
337 of the file time to determine whether to sync */
338 if (always_checksum && S_ISREG(st->st_mode)) {
339 char sum[MD4_SUM_LENGTH];
340 file_checksum(fname,sum,st->st_size);
341 return (memcmp(sum,file->sum,csum_length) == 0);
348 return (st->st_mtime == file->modtime);
352 /* use a larger block size for really big files */
353 int adapt_block_size(struct file_struct *file, int bsize)
355 int ret = file->length / (10000); /* rough heuristic */
356 ret = ret & ~15; /* multiple of 16 */
357 if (ret < bsize) ret = bsize;
358 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
362 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
366 struct map_struct *buf;
367 struct sum_struct *s;
369 struct file_struct *file = flist->files[i];
372 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
374 statret = link_stat(fname,&st);
376 if (S_ISDIR(file->mode)) {
378 if (statret == 0 && !S_ISDIR(st.st_mode)) {
379 if (do_unlink(fname) != 0) {
380 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
385 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
386 if (!(relative_paths && errno==ENOENT &&
387 create_directory_path(fname)==0 &&
388 do_mkdir(fname,file->mode)==0)) {
389 rprintf(FERROR,"mkdir %s : %s (2)\n",
390 fname,strerror(errno));
393 if (set_perms(fname,file,NULL,0) && verbose)
394 rprintf(FINFO,"%s/\n",fname);
398 if (preserve_links && S_ISLNK(file->mode)) {
400 char lnk[MAXPATHLEN];
403 l = readlink(fname,lnk,MAXPATHLEN-1);
406 if (strcmp(lnk,file->link) == 0) {
407 set_perms(fname,file,&st,1);
413 if (do_symlink(file->link,fname) != 0) {
414 rprintf(FERROR,"link %s -> %s : %s\n",
415 fname,file->link,strerror(errno));
417 set_perms(fname,file,NULL,0);
419 rprintf(FINFO,"%s -> %s\n",
427 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
429 st.st_mode != file->mode ||
430 st.st_rdev != file->rdev) {
433 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
434 fname,(int)file->mode,(int)file->rdev);
435 if (do_mknod(fname,file->mode,file->rdev) != 0) {
436 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
438 set_perms(fname,file,NULL,0);
440 rprintf(FINFO,"%s\n",fname);
443 set_perms(fname,file,&st,1);
449 if (preserve_hard_links && check_hard_link(file)) {
451 rprintf(FINFO,"%s is a hard link\n",f_name(file));
455 if (!S_ISREG(file->mode)) {
456 rprintf(FINFO,"skipping non-regular file %s\n",fname);
461 if (errno == ENOENT) {
463 if (!dry_run) send_sums(NULL,f_out);
466 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
471 if (!S_ISREG(st.st_mode)) {
472 if (delete_file(fname) != 0) {
476 /* now pretend the file didn't exist */
478 if (!dry_run) send_sums(NULL,f_out);
482 if (update_only && st.st_mtime > file->modtime) {
484 rprintf(FINFO,"%s is newer\n",fname);
488 if (skip_file(fname, file, &st)) {
489 set_perms(fname,file,&st,1);
500 send_sums(NULL,f_out);
505 fd = open(fname,O_RDONLY);
508 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
509 rprintf(FERROR,"skipping %s\n",fname);
513 if (st.st_size > 0) {
514 buf = map_file(fd,st.st_size);
520 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
522 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
525 rprintf(FINFO,"sending sums for %d\n",i);
532 if (buf) unmap_file(buf);
539 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
541 int i,n,remainder,len,count;
545 static char file_sum1[MD4_SUM_LENGTH];
546 static char file_sum2[MD4_SUM_LENGTH];
549 count = read_int(f_in);
551 remainder = read_int(f_in);
555 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
558 rprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
562 if (fd != -1 && write_file(fd,data,i) != i) {
563 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
571 if (i == count-1 && remainder != 0)
575 rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
576 i,len,(int)offset2,(int)offset);
578 map = map_ptr(buf,offset2,len);
583 if (fd != -1 && write_file(fd,map,len) != len) {
584 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
591 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
592 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
598 if (remote_version >= 14) {
599 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
601 rprintf(FINFO,"got file_sum\n");
602 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
609 static void delete_one(struct file_struct *f)
611 if (!S_ISDIR(f->mode)) {
612 if (do_unlink(f_name(f)) != 0) {
613 rprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
614 } else if (verbose) {
615 rprintf(FINFO,"deleting %s\n",f_name(f));
618 if (do_rmdir(f_name(f)) != 0) {
619 if (errno != ENOTEMPTY && errno != EEXIST)
620 rprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
621 } else if (verbose) {
622 rprintf(FINFO,"deleting directory %s\n",f_name(f));
629 static struct delete_list {
633 static int dlist_len, dlist_alloc_len;
635 static void add_delete_entry(struct file_struct *file)
637 if (dlist_len == dlist_alloc_len) {
638 dlist_alloc_len += 1024;
640 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
642 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
644 if (!delete_list) out_of_memory("add_delete_entry");
647 delete_list[dlist_len].dev = file->dev;
648 delete_list[dlist_len].inode = file->inode;
652 rprintf(FINFO,"added %s to delete list\n", f_name(file));
655 /* yuck! This function wouldn't have been necessary if I had the sorting
656 algorithm right. Unfortunately fixing the sorting algorithm would introduce
657 a backward incompatibility as file list indexes are sent over the link.
659 static int delete_already_done(struct file_list *flist,int j)
664 if (link_stat(f_name(flist->files[j]), &st)) return 1;
666 for (i=0;i<dlist_len;i++) {
667 if (st.st_ino == delete_list[i].inode &&
668 st.st_dev == delete_list[i].dev)
676 /* this deletes any files on the receiving side that are not present
677 on the sending side. For version 1.6.4 I have changed the behaviour
678 to match more closely what most people seem to expect of this option */
679 static void delete_files(struct file_list *flist)
681 struct file_list *local_file_list;
689 rprintf(FINFO,"IO error encountered - skipping file deletion\n");
693 for (j=0;j<flist->count;j++) {
694 if (!S_ISDIR(flist->files[j]->mode) ||
695 !(flist->files[j]->flags & FLAG_DELETE)) continue;
697 if (delete_already_done(flist, j)) continue;
699 name = strdup(f_name(flist->files[j]));
701 if (!(local_file_list = send_file_list(-1,1,&name))) {
707 rprintf(FINFO,"deleting in %s\n", name);
709 for (i=local_file_list->count-1;i>=0;i--) {
710 if (!local_file_list->files[i]->basename) continue;
711 if (S_ISDIR(local_file_list->files[i]->mode))
712 add_delete_entry(local_file_list->files[i]);
713 if (-1 == flist_find(flist,local_file_list->files[i])) {
714 delete_one(local_file_list->files[i]);
717 flist_free(local_file_list);
722 static char *cleanup_fname;
724 void exit_cleanup(int code)
727 do_unlink(cleanup_fname);
728 signal(SIGUSR1, SIG_IGN);
741 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
746 char fnametmp[MAXPATHLEN];
747 struct map_struct *buf;
749 struct file_struct *file;
754 rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
757 if (recurse && delete_mode && !local_name && flist->count>0) {
765 if (phase==0 && remote_version >= 13) {
767 csum_length = SUM_LENGTH;
769 rprintf(FINFO,"recv_files phase=%d\n",phase);
777 file = flist->files[i];
778 fname = f_name(file);
784 if (!am_server && verbose)
785 printf("%s\n",fname);
790 rprintf(FINFO,"recv_files(%s)\n",fname);
793 fd1 = open(fname,O_RDONLY);
795 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
796 rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
797 receive_data(f_in,NULL,-1,NULL);
802 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
803 rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
804 receive_data(f_in,NULL,-1,NULL);
809 if (fd1 != -1 && st.st_size > 0) {
810 buf = map_file(fd1,st.st_size);
812 rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
820 f = strrchr(fname,'/');
825 if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) {
826 rprintf(FERROR,"filename too long\n");
827 if (buf) unmap_file(buf);
831 slprintf(fnametmp,sizeof(fnametmp)-1, "%s/.%s.XXXXXX",tmpdir,f);
833 char *f = strrchr(fname,'/');
835 if (strlen(fname)+9 > MAXPATHLEN) {
836 rprintf(FERROR,"filename too long\n");
837 if (buf) unmap_file(buf);
844 slprintf(fnametmp,sizeof(fnametmp)-1,"%s/.%s.XXXXXX",fname,f+1);
847 slprintf(fnametmp,sizeof(fnametmp)-1,".%s.XXXXXX",fname);
850 if (NULL == do_mktemp(fnametmp)) {
851 rprintf(FERROR,"mktemp %s failed\n",fnametmp);
852 receive_data(f_in,buf,-1,NULL);
853 if (buf) unmap_file(buf);
857 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
858 if (fd2 == -1 && relative_paths && errno == ENOENT &&
859 create_directory_path(fnametmp) == 0) {
860 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
863 rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
864 receive_data(f_in,buf,-1,NULL);
865 if (buf) unmap_file(buf);
870 cleanup_fname = fnametmp;
872 if (!am_server && verbose)
873 printf("%s\n",fname);
876 recv_ok = receive_data(f_in,buf,fd2,fname);
878 if (buf) unmap_file(buf);
885 rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
888 char fnamebak[MAXPATHLEN];
889 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
890 rprintf(FERROR,"backup filename too long\n");
893 slprintf(fnamebak,sizeof(fnamebak)-1,"%s%s",fname,backup_suffix);
894 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
895 rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
900 /* move tmp file over real file */
901 if (do_rename(fnametmp,fname) != 0) {
902 if (errno == EXDEV) {
903 /* rename failed on cross-filesystem link.
904 Copy the file instead. */
905 if (copy_file(fnametmp,fname, file->mode)) {
906 rprintf(FERROR,"copy %s -> %s : %s\n",
907 fnametmp,fname,strerror(errno));
909 set_perms(fname,file,NULL,0);
913 rprintf(FERROR,"rename %s -> %s : %s\n",
914 fnametmp,fname,strerror(errno));
918 set_perms(fname,file,NULL,0);
921 cleanup_fname = NULL;
925 if (csum_length == SUM_LENGTH) {
926 rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
930 rprintf(FINFO,"redoing %s(%d)\n",fname,i);
936 if (preserve_hard_links)
937 do_hard_links(flist);
939 /* now we need to fix any directory permissions that were
940 modified during the transfer */
941 for (i = 0; i < flist->count; i++) {
942 struct file_struct *file = flist->files[i];
943 if (!file->basename || !S_ISDIR(file->mode)) continue;
944 recv_generator(f_name(file),flist,i,-1);
948 rprintf(FINFO,"recv_files finished\n");
955 void send_files(struct file_list *flist,int f_out,int f_in)
958 struct sum_struct *s;
959 struct map_struct *buf;
961 char fname[MAXPATHLEN];
963 struct file_struct *file;
968 rprintf(FINFO,"send_files starting\n");
970 setup_nonblocking(f_in,f_out);
975 if (phase==0 && remote_version >= 13) {
977 csum_length = SUM_LENGTH;
981 rprintf(FINFO,"send_files phase=%d\n",phase);
987 file = flist->files[i];
991 strlcpy(fname,file->basedir,MAXPATHLEN-1);
992 if (strlen(fname) == MAXPATHLEN-1) {
994 rprintf(FERROR, "send_files failed on long-named directory %s\n",
998 strlcat(fname,"/",MAXPATHLEN-1);
999 offset = strlen(file->basedir)+1;
1001 strlcat(fname,f_name(file),MAXPATHLEN-strlen(fname));
1004 rprintf(FINFO,"send_files(%d,%s)\n",i,fname);
1007 if (!am_server && verbose)
1008 printf("%s\n",fname);
1013 s = receive_sums(f_in);
1016 rprintf(FERROR,"receive_sums failed\n");
1020 fd = open(fname,O_RDONLY);
1023 rprintf(FERROR,"send_files failed to open %s: %s\n",
1024 fname,strerror(errno));
1029 /* map the local file */
1030 if (do_fstat(fd,&st) != 0) {
1032 rprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1038 if (st.st_size > 0) {
1039 buf = map_file(fd,st.st_size);
1045 rprintf(FINFO,"send_files mapped %s of size %d\n",
1046 fname,(int)st.st_size);
1050 write_int(f_out,s->count);
1051 write_int(f_out,s->n);
1052 write_int(f_out,s->remainder);
1055 rprintf(FINFO,"calling match_sums %s\n",fname);
1057 if (!am_server && verbose)
1058 printf("%s\n",fname+offset);
1060 match_sums(f_out,s,buf,st.st_size);
1063 if (buf) unmap_file(buf);
1069 rprintf(FINFO,"sender finished %s\n",fname);
1073 rprintf(FINFO,"send files finished\n");
1077 write_int(f_out,-1);
1083 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1089 rprintf(FINFO,"generator starting pid=%d count=%d\n",
1090 (int)getpid(),flist->count);
1092 for (i = 0; i < flist->count; i++) {
1093 struct file_struct *file = flist->files[i];
1094 mode_t saved_mode = file->mode;
1095 if (!file->basename) continue;
1097 /* we need to ensure that any directories we create have writeable
1098 permissions initially so that we can create the files within
1099 them. This is then fixed after the files are transferred */
1100 if (!am_root && S_ISDIR(file->mode)) {
1101 file->mode |= S_IWUSR; /* user write */
1104 recv_generator(local_name?local_name:f_name(file),
1107 file->mode = saved_mode;
1111 csum_length = SUM_LENGTH;
1115 rprintf(FINFO,"generate_files phase=%d\n",phase);
1120 /* we expect to just sit around now, so don't exit on a timeout. If we
1121 really get a timeout then the other process should exit */
1124 if (remote_version >= 13) {
1125 /* in newer versions of the protocol the files can cycle through
1126 the system more than once to catch initial checksum errors */
1127 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1128 struct file_struct *file = flist->files[i];
1129 recv_generator(local_name?local_name:f_name(file),
1135 rprintf(FINFO,"generate_files phase=%d\n",phase);
1143 rprintf(FINFO,"generator wrote %ld\n",(long)write_total());