2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 rprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 rprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 rprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 strncpy(buf, fname, (MAXPATHLEN-strlen(dname))-2);
117 buf[MAXPATHLEN-1] = 0;
119 rprintf(FINFO,"deleting %s\n", buf);
120 if (delete_file(buf) != 0) {
128 if (do_rmdir(fname) != 0) {
129 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
137 send a sums struct down a fd
139 static void send_sums(struct sum_struct *s,int f_out)
143 /* tell the other guy how many we are going to be doing and how many
144 bytes there are in the last chunk */
145 write_int(f_out,s?s->count:0);
146 write_int(f_out,s?s->n:block_size);
147 write_int(f_out,s?s->remainder:0);
149 for (i=0;i<s->count;i++) {
150 write_int(f_out,s->sums[i].sum1);
151 write_buf(f_out,s->sums[i].sum2,csum_length);
158 generate a stream of signatures/checksums that describe a buffer
160 generate approximately one checksum every n bytes
162 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
165 struct sum_struct *s;
168 int remainder = (len%block_len);
171 count = (len+(block_len-1))/block_len;
173 s = (struct sum_struct *)malloc(sizeof(*s));
174 if (!s) out_of_memory("generate_sums");
177 s->remainder = remainder;
187 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
188 s->count,s->remainder,s->n,(int)s->flength);
190 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
191 if (!s->sums) out_of_memory("generate_sums");
193 for (i=0;i<count;i++) {
195 char *map = map_ptr(buf,offset,n1);
197 s->sums[i].sum1 = get_checksum1(map,n1);
198 get_checksum2(map,n1,s->sums[i].sum2);
200 s->sums[i].offset = offset;
205 rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
206 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
217 receive the checksums for a buffer
219 static struct sum_struct *receive_sums(int f)
221 struct sum_struct *s;
225 s = (struct sum_struct *)malloc(sizeof(*s));
226 if (!s) out_of_memory("receive_sums");
228 s->count = read_int(f);
230 s->remainder = read_int(f);
234 rprintf(FINFO,"count=%d n=%d rem=%d\n",
235 s->count,s->n,s->remainder);
240 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
241 if (!s->sums) out_of_memory("receive_sums");
243 for (i=0;i<s->count;i++) {
244 s->sums[i].sum1 = read_int(f);
245 read_buf(f,s->sums[i].sum2,csum_length);
247 s->sums[i].offset = offset;
250 if (i == s->count-1 && s->remainder != 0) {
251 s->sums[i].len = s->remainder;
253 s->sums[i].len = s->n;
255 offset += s->sums[i].len;
258 rprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
259 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
268 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
274 if (dry_run) return 0;
277 if (link_stat(fname,&st2) != 0) {
278 rprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
284 if (preserve_times && !S_ISLNK(st->st_mode) &&
285 st->st_mtime != file->modtime) {
287 if (set_modtime(fname,file->modtime) != 0) {
288 rprintf(FERROR,"failed to set times on %s : %s\n",
289 fname,strerror(errno));
295 if (preserve_perms && !S_ISLNK(st->st_mode) &&
296 st->st_mode != file->mode) {
298 if (do_chmod(fname,file->mode) != 0) {
299 rprintf(FERROR,"failed to set permissions on %s : %s\n",
300 fname,strerror(errno));
306 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
307 (preserve_gid && st->st_gid != file->gid)) {
309 (am_root&&preserve_uid)?file->uid:-1,
310 preserve_gid?file->gid:-1) != 0) {
311 if (preserve_uid && st->st_uid != file->uid)
313 if (verbose>1 || preserve_uid)
314 rprintf(FERROR,"chown %s : %s\n",
315 fname,strerror(errno));
321 if (verbose > 1 && report) {
323 rprintf(FINFO,"%s\n",fname);
325 rprintf(FINFO,"%s is uptodate\n",fname);
331 /* choose whether to skip a particular file */
332 static int skip_file(char *fname,
333 struct file_struct *file, STRUCT_STAT *st)
335 if (st->st_size != file->length) {
339 /* if always checksum is set then we use the checksum instead
340 of the file time to determine whether to sync */
341 if (always_checksum && S_ISREG(st->st_mode)) {
342 char sum[MD4_SUM_LENGTH];
343 file_checksum(fname,sum,st->st_size);
344 return (memcmp(sum,file->sum,csum_length) == 0);
351 return (st->st_mtime == file->modtime);
355 /* use a larger block size for really big files */
356 int adapt_block_size(struct file_struct *file, int bsize)
358 int ret = file->length / (10000); /* rough heuristic */
359 ret = ret & ~15; /* multiple of 16 */
360 if (ret < bsize) ret = bsize;
361 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
365 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
369 struct map_struct *buf;
370 struct sum_struct *s;
372 struct file_struct *file = flist->files[i];
375 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
377 statret = link_stat(fname,&st);
379 if (S_ISDIR(file->mode)) {
381 if (statret == 0 && !S_ISDIR(st.st_mode)) {
382 if (do_unlink(fname) != 0) {
383 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
388 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
389 if (!(relative_paths && errno==ENOENT &&
390 create_directory_path(fname)==0 &&
391 do_mkdir(fname,file->mode)==0)) {
392 rprintf(FERROR,"mkdir %s : %s (2)\n",
393 fname,strerror(errno));
396 if (set_perms(fname,file,NULL,0) && verbose)
397 rprintf(FINFO,"%s/\n",fname);
401 if (preserve_links && S_ISLNK(file->mode)) {
403 char lnk[MAXPATHLEN];
406 l = readlink(fname,lnk,MAXPATHLEN-1);
409 if (strcmp(lnk,file->link) == 0) {
410 set_perms(fname,file,&st,1);
416 if (do_symlink(file->link,fname) != 0) {
417 rprintf(FERROR,"link %s -> %s : %s\n",
418 fname,file->link,strerror(errno));
420 set_perms(fname,file,NULL,0);
422 rprintf(FINFO,"%s -> %s\n",
430 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
432 st.st_mode != file->mode ||
433 st.st_rdev != file->rdev) {
436 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
437 fname,(int)file->mode,(int)file->rdev);
438 if (do_mknod(fname,file->mode,file->rdev) != 0) {
439 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
441 set_perms(fname,file,NULL,0);
443 rprintf(FINFO,"%s\n",fname);
446 set_perms(fname,file,&st,1);
452 if (preserve_hard_links && check_hard_link(file)) {
454 rprintf(FINFO,"%s is a hard link\n",f_name(file));
458 if (!S_ISREG(file->mode)) {
459 rprintf(FINFO,"skipping non-regular file %s\n",fname);
464 if (errno == ENOENT) {
466 if (!dry_run) send_sums(NULL,f_out);
469 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
474 if (!S_ISREG(st.st_mode)) {
475 if (delete_file(fname) != 0) {
479 /* now pretend the file didn't exist */
481 if (!dry_run) send_sums(NULL,f_out);
485 if (update_only && st.st_mtime > file->modtime) {
487 rprintf(FINFO,"%s is newer\n",fname);
491 if (skip_file(fname, file, &st)) {
492 set_perms(fname,file,&st,1);
503 send_sums(NULL,f_out);
508 fd = open(fname,O_RDONLY);
511 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
512 rprintf(FERROR,"skipping %s\n",fname);
516 if (st.st_size > 0) {
517 buf = map_file(fd,st.st_size);
523 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
525 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
528 rprintf(FINFO,"sending sums for %d\n",i);
535 if (buf) unmap_file(buf);
542 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
544 int i,n,remainder,len,count;
548 static char file_sum1[MD4_SUM_LENGTH];
549 static char file_sum2[MD4_SUM_LENGTH];
552 count = read_int(f_in);
554 remainder = read_int(f_in);
558 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
561 rprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
565 if (fd != -1 && write_file(fd,data,i) != i) {
566 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
574 if (i == count-1 && remainder != 0)
578 rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
579 i,len,(int)offset2,(int)offset);
581 map = map_ptr(buf,offset2,len);
586 if (fd != -1 && write_file(fd,map,len) != len) {
587 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
594 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
595 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
601 if (remote_version >= 14) {
602 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
604 rprintf(FINFO,"got file_sum\n");
605 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
612 static void delete_one(struct file_struct *f)
614 if (!S_ISDIR(f->mode)) {
615 if (do_unlink(f_name(f)) != 0) {
616 rprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
617 } else if (verbose) {
618 rprintf(FINFO,"deleting %s\n",f_name(f));
621 if (do_rmdir(f_name(f)) != 0) {
622 if (errno != ENOTEMPTY && errno != EEXIST)
623 rprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
624 } else if (verbose) {
625 rprintf(FINFO,"deleting directory %s\n",f_name(f));
632 static struct delete_list {
636 static int dlist_len, dlist_alloc_len;
638 static void add_delete_entry(struct file_struct *file)
640 if (dlist_len == dlist_alloc_len) {
641 dlist_alloc_len += 1024;
643 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
645 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
647 if (!delete_list) out_of_memory("add_delete_entry");
650 delete_list[dlist_len].dev = file->dev;
651 delete_list[dlist_len].inode = file->inode;
655 rprintf(FINFO,"added %s to delete list\n", f_name(file));
658 /* yuck! This function wouldn't have been necessary if I had the sorting
659 algorithm right. Unfortunately fixing the sorting algorithm would introduce
660 a backward incompatibility as file list indexes are sent over the link.
662 static int delete_already_done(struct file_list *flist,int j)
667 if (link_stat(f_name(flist->files[j]), &st)) return 1;
669 for (i=0;i<dlist_len;i++) {
670 if (st.st_ino == delete_list[i].inode &&
671 st.st_dev == delete_list[i].dev)
679 /* this deletes any files on the receiving side that are not present
680 on the sending side. For version 1.6.4 I have changed the behaviour
681 to match more closely what most people seem to expect of this option */
682 static void delete_files(struct file_list *flist)
684 struct file_list *local_file_list;
692 rprintf(FINFO,"IO error encountered - skipping file deletion\n");
696 for (j=0;j<flist->count;j++) {
697 if (!S_ISDIR(flist->files[j]->mode) ||
698 !(flist->files[j]->flags & FLAG_DELETE)) continue;
700 if (delete_already_done(flist, j)) continue;
702 name = strdup(f_name(flist->files[j]));
704 if (!(local_file_list = send_file_list(-1,1,&name))) {
710 rprintf(FINFO,"deleting in %s\n", name);
712 for (i=local_file_list->count-1;i>=0;i--) {
713 if (!local_file_list->files[i]->basename) continue;
714 if (S_ISDIR(local_file_list->files[i]->mode))
715 add_delete_entry(local_file_list->files[i]);
716 if (-1 == flist_find(flist,local_file_list->files[i])) {
717 delete_one(local_file_list->files[i]);
720 flist_free(local_file_list);
725 static char *cleanup_fname;
727 void exit_cleanup(int code)
730 do_unlink(cleanup_fname);
731 signal(SIGUSR1, SIG_IGN);
744 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
749 char fnametmp[MAXPATHLEN];
750 struct map_struct *buf;
752 struct file_struct *file;
757 rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
760 if (recurse && delete_mode && !local_name && flist->count>0) {
768 if (phase==0 && remote_version >= 13) {
770 csum_length = SUM_LENGTH;
772 rprintf(FINFO,"recv_files phase=%d\n",phase);
780 file = flist->files[i];
781 fname = f_name(file);
787 if (!am_server && verbose)
788 printf("%s\n",fname);
793 rprintf(FINFO,"recv_files(%s)\n",fname);
796 fd1 = open(fname,O_RDONLY);
798 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
799 rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
800 receive_data(f_in,NULL,-1,NULL);
805 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
806 rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
807 receive_data(f_in,NULL,-1,NULL);
812 if (fd1 != -1 && st.st_size > 0) {
813 buf = map_file(fd1,st.st_size);
815 rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
821 if (strlen(fname) > (MAXPATHLEN-8)) {
822 rprintf(FERROR,"filename too long\n");
823 if (buf) unmap_file(buf);
829 f = strrchr(fname,'/');
834 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
836 sprintf(fnametmp,"%s.XXXXXX",fname);
838 if (NULL == do_mktemp(fnametmp)) {
839 rprintf(FERROR,"mktemp %s failed\n",fnametmp);
840 receive_data(f_in,buf,-1,NULL);
841 if (buf) unmap_file(buf);
845 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
846 if (fd2 == -1 && relative_paths && errno == ENOENT &&
847 create_directory_path(fnametmp) == 0) {
848 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
851 rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
852 receive_data(f_in,buf,-1,NULL);
853 if (buf) unmap_file(buf);
858 cleanup_fname = fnametmp;
860 if (!am_server && verbose)
861 printf("%s\n",fname);
864 recv_ok = receive_data(f_in,buf,fd2,fname);
866 if (buf) unmap_file(buf);
873 rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
876 char fnamebak[MAXPATHLEN];
877 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
878 rprintf(FERROR,"backup filename too long\n");
881 sprintf(fnamebak,"%s%s",fname,backup_suffix);
882 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
883 rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
888 /* move tmp file over real file */
889 if (do_rename(fnametmp,fname) != 0) {
890 if (errno == EXDEV) {
891 /* rename failed on cross-filesystem link.
892 Copy the file instead. */
893 if (copy_file(fnametmp,fname, file->mode)) {
894 rprintf(FERROR,"copy %s -> %s : %s\n",
895 fnametmp,fname,strerror(errno));
897 set_perms(fname,file,NULL,0);
901 rprintf(FERROR,"rename %s -> %s : %s\n",
902 fnametmp,fname,strerror(errno));
906 set_perms(fname,file,NULL,0);
909 cleanup_fname = NULL;
913 if (csum_length == SUM_LENGTH) {
914 rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
918 rprintf(FINFO,"redoing %s(%d)\n",fname,i);
924 if (preserve_hard_links)
925 do_hard_links(flist);
927 /* now we need to fix any directory permissions that were
928 modified during the transfer */
929 for (i = 0; i < flist->count; i++) {
930 struct file_struct *file = flist->files[i];
931 if (!file->basename || !S_ISDIR(file->mode)) continue;
932 recv_generator(f_name(file),flist,i,-1);
936 rprintf(FINFO,"recv_files finished\n");
943 void send_files(struct file_list *flist,int f_out,int f_in)
946 struct sum_struct *s;
947 struct map_struct *buf;
949 char fname[MAXPATHLEN];
951 struct file_struct *file;
956 rprintf(FINFO,"send_files starting\n");
958 setup_nonblocking(f_in,f_out);
963 if (phase==0 && remote_version >= 13) {
965 csum_length = SUM_LENGTH;
969 rprintf(FINFO,"send_files phase=%d\n",phase);
975 file = flist->files[i];
979 strncpy(fname,file->basedir,MAXPATHLEN-1);
980 fname[MAXPATHLEN-1] = 0;
981 if (strlen(fname) == MAXPATHLEN-1) {
983 rprintf(FERROR, "send_files failed on long-named directory %s\n",
988 offset = strlen(file->basedir)+1;
990 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
993 rprintf(FINFO,"send_files(%d,%s)\n",i,fname);
996 if (!am_server && verbose)
997 printf("%s\n",fname);
1002 s = receive_sums(f_in);
1005 rprintf(FERROR,"receive_sums failed\n");
1009 fd = open(fname,O_RDONLY);
1012 rprintf(FERROR,"send_files failed to open %s: %s\n",
1013 fname,strerror(errno));
1018 /* map the local file */
1019 if (do_fstat(fd,&st) != 0) {
1021 rprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1027 if (st.st_size > 0) {
1028 buf = map_file(fd,st.st_size);
1034 rprintf(FINFO,"send_files mapped %s of size %d\n",
1035 fname,(int)st.st_size);
1039 write_int(f_out,s->count);
1040 write_int(f_out,s->n);
1041 write_int(f_out,s->remainder);
1044 rprintf(FINFO,"calling match_sums %s\n",fname);
1046 if (!am_server && verbose)
1047 printf("%s\n",fname+offset);
1049 match_sums(f_out,s,buf,st.st_size);
1052 if (buf) unmap_file(buf);
1058 rprintf(FINFO,"sender finished %s\n",fname);
1062 rprintf(FINFO,"send files finished\n");
1066 write_int(f_out,-1);
1072 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1078 rprintf(FINFO,"generator starting pid=%d count=%d\n",
1079 (int)getpid(),flist->count);
1081 for (i = 0; i < flist->count; i++) {
1082 struct file_struct *file = flist->files[i];
1083 mode_t saved_mode = file->mode;
1084 if (!file->basename) continue;
1086 /* we need to ensure that any directories we create have writeable
1087 permissions initially so that we can create the files within
1088 them. This is then fixed after the files are transferred */
1089 if (!am_root && S_ISDIR(file->mode)) {
1090 file->mode |= S_IWUSR; /* user write */
1093 recv_generator(local_name?local_name:f_name(file),
1096 file->mode = saved_mode;
1100 csum_length = SUM_LENGTH;
1104 rprintf(FINFO,"generate_files phase=%d\n",phase);
1109 /* we expect to just sit around now, so don't exit on a timeout. If we
1110 really get a timeout then the other process should exit */
1113 if (remote_version >= 13) {
1114 /* in newer versions of the protocol the files can cycle through
1115 the system more than once to catch initial checksum errors */
1116 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1117 struct file_struct *file = flist->files[i];
1118 recv_generator(local_name?local_name:f_name(file),
1124 rprintf(FINFO,"generate_files phase=%d\n",phase);
1132 rprintf(FINFO,"generator wrote %ld\n",(long)write_total());