2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 fprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 fprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 fprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 strncpy(buf, fname, (MAXPATHLEN-strlen(dname))-2);
117 buf[MAXPATHLEN-1] = 0;
119 fprintf(FINFO,"deleting %s\n", buf);
120 if (delete_file(buf) != 0) {
128 if (do_rmdir(fname) != 0) {
129 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
137 send a sums struct down a fd
139 static void send_sums(struct sum_struct *s,int f_out)
143 /* tell the other guy how many we are going to be doing and how many
144 bytes there are in the last chunk */
145 write_int(f_out,s?s->count:0);
146 write_int(f_out,s?s->n:block_size);
147 write_int(f_out,s?s->remainder:0);
149 for (i=0;i<s->count;i++) {
150 write_int(f_out,s->sums[i].sum1);
151 write_buf(f_out,s->sums[i].sum2,csum_length);
158 generate a stream of signatures/checksums that describe a buffer
160 generate approximately one checksum every n bytes
162 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
165 struct sum_struct *s;
168 int remainder = (len%block_len);
171 count = (len+(block_len-1))/block_len;
173 s = (struct sum_struct *)malloc(sizeof(*s));
174 if (!s) out_of_memory("generate_sums");
177 s->remainder = remainder;
187 fprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
188 s->count,s->remainder,s->n,(int)s->flength);
190 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
191 if (!s->sums) out_of_memory("generate_sums");
193 for (i=0;i<count;i++) {
195 char *map = map_ptr(buf,offset,n1);
197 s->sums[i].sum1 = get_checksum1(map,n1);
198 get_checksum2(map,n1,s->sums[i].sum2);
200 s->sums[i].offset = offset;
205 fprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
206 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
217 receive the checksums for a buffer
219 static struct sum_struct *receive_sums(int f)
221 struct sum_struct *s;
225 s = (struct sum_struct *)malloc(sizeof(*s));
226 if (!s) out_of_memory("receive_sums");
228 s->count = read_int(f);
230 s->remainder = read_int(f);
234 fprintf(FINFO,"count=%d n=%d rem=%d\n",
235 s->count,s->n,s->remainder);
240 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
241 if (!s->sums) out_of_memory("receive_sums");
243 for (i=0;i<s->count;i++) {
244 s->sums[i].sum1 = read_int(f);
245 read_buf(f,s->sums[i].sum2,csum_length);
247 s->sums[i].offset = offset;
250 if (i == s->count-1 && s->remainder != 0) {
251 s->sums[i].len = s->remainder;
253 s->sums[i].len = s->n;
255 offset += s->sums[i].len;
258 fprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
259 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
268 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
274 if (dry_run) return 0;
277 if (link_stat(fname,&st2) != 0) {
278 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
284 if (preserve_times && !S_ISLNK(st->st_mode) &&
285 st->st_mtime != file->modtime) {
287 if (set_modtime(fname,file->modtime) != 0) {
288 fprintf(FERROR,"failed to set times on %s : %s\n",
289 fname,strerror(errno));
295 if (preserve_perms && !S_ISLNK(st->st_mode) &&
296 st->st_mode != file->mode) {
298 if (do_chmod(fname,file->mode) != 0) {
299 fprintf(FERROR,"failed to set permissions on %s : %s\n",
300 fname,strerror(errno));
306 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
307 (preserve_gid && st->st_gid != file->gid)) {
309 (am_root&&preserve_uid)?file->uid:-1,
310 preserve_gid?file->gid:-1) != 0) {
311 if (preserve_uid && st->st_uid != file->uid)
313 if (verbose>1 || preserve_uid)
314 fprintf(FERROR,"chown %s : %s\n",
315 fname,strerror(errno));
321 if (verbose > 1 && report) {
323 fprintf(FINFO,"%s\n",fname);
325 fprintf(FINFO,"%s is uptodate\n",fname);
331 /* choose whether to skip a particular file */
332 static int skip_file(char *fname,
333 struct file_struct *file, STRUCT_STAT *st)
335 if (st->st_size != file->length) {
339 /* if always checksum is set then we use the checksum instead
340 of the file time to determine whether to sync */
341 if (always_checksum && S_ISREG(st->st_mode)) {
342 char sum[MD4_SUM_LENGTH];
343 file_checksum(fname,sum,st->st_size);
344 return (memcmp(sum,file->sum,csum_length) == 0);
351 return (st->st_mtime == file->modtime);
355 /* use a larger block size for really big files */
356 int adapt_block_size(struct file_struct *file, int bsize)
358 int ret = file->length / (10000); /* rough heuristic */
359 ret = ret & ~15; /* multiple of 16 */
360 if (ret < bsize) ret = bsize;
361 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
365 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
369 struct map_struct *buf;
370 struct sum_struct *s;
372 struct file_struct *file = flist->files[i];
375 fprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
377 statret = link_stat(fname,&st);
379 if (S_ISDIR(file->mode)) {
381 if (statret == 0 && !S_ISDIR(st.st_mode)) {
382 if (do_unlink(fname) != 0) {
383 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
388 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
389 if (!(relative_paths && errno==ENOENT &&
390 create_directory_path(fname)==0 &&
391 do_mkdir(fname,file->mode)==0)) {
392 fprintf(FERROR,"mkdir %s : %s (2)\n",
393 fname,strerror(errno));
396 if (set_perms(fname,file,NULL,0) && verbose)
397 fprintf(FINFO,"%s/\n",fname);
401 if (preserve_links && S_ISLNK(file->mode)) {
403 char lnk[MAXPATHLEN];
406 l = readlink(fname,lnk,MAXPATHLEN-1);
409 if (strcmp(lnk,file->link) == 0) {
410 set_perms(fname,file,&st,1);
416 if (do_symlink(file->link,fname) != 0) {
417 fprintf(FERROR,"link %s -> %s : %s\n",
418 fname,file->link,strerror(errno));
420 set_perms(fname,file,NULL,0);
422 fprintf(FINFO,"%s -> %s\n",
430 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
432 st.st_mode != file->mode ||
433 st.st_rdev != file->rdev) {
436 fprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
437 fname,(int)file->mode,(int)file->rdev);
438 if (do_mknod(fname,file->mode,file->rdev) != 0) {
439 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
441 set_perms(fname,file,NULL,0);
443 fprintf(FINFO,"%s\n",fname);
446 set_perms(fname,file,&st,1);
452 if (preserve_hard_links && check_hard_link(file)) {
454 fprintf(FINFO,"%s is a hard link\n",f_name(file));
458 if (!S_ISREG(file->mode)) {
459 fprintf(FINFO,"skipping non-regular file %s\n",fname);
464 if (errno == ENOENT) {
466 if (!dry_run) send_sums(NULL,f_out);
469 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
474 if (!S_ISREG(st.st_mode)) {
475 if (delete_file(fname) != 0) {
479 /* now pretend the file didn't exist */
481 if (!dry_run) send_sums(NULL,f_out);
485 if (update_only && st.st_mtime > file->modtime) {
487 fprintf(FINFO,"%s is newer\n",fname);
491 if (skip_file(fname, file, &st)) {
492 set_perms(fname,file,&st,1);
503 send_sums(NULL,f_out);
508 fd = open(fname,O_RDONLY);
511 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
512 fprintf(FERROR,"skipping %s\n",fname);
516 if (st.st_size > 0) {
517 buf = map_file(fd,st.st_size);
523 fprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
525 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
528 fprintf(FINFO,"sending sums for %d\n",i);
535 if (buf) unmap_file(buf);
542 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
544 int i,n,remainder,len,count;
548 static char file_sum1[MD4_SUM_LENGTH];
549 static char file_sum2[MD4_SUM_LENGTH];
552 count = read_int(f_in);
554 remainder = read_int(f_in);
558 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
561 fprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
565 if (fd != -1 && write_file(fd,data,i) != i) {
566 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
574 if (i == count-1 && remainder != 0)
578 fprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
579 i,len,(int)offset2,(int)offset);
581 map = map_ptr(buf,offset2,len);
586 if (fd != -1 && write_file(fd,map,len) != len) {
587 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
594 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
595 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
601 if (remote_version >= 14) {
602 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
604 fprintf(FINFO,"got file_sum\n");
605 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
612 static void delete_one(struct file_struct *f)
614 if (!S_ISDIR(f->mode)) {
615 if (do_unlink(f_name(f)) != 0) {
616 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
617 } else if (verbose) {
618 fprintf(FINFO,"deleting %s\n",f_name(f));
621 if (do_rmdir(f_name(f)) != 0) {
622 if (errno != ENOTEMPTY && errno != EEXIST)
623 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
624 } else if (verbose) {
625 fprintf(FINFO,"deleting directory %s\n",f_name(f));
632 static struct delete_list {
636 static int dlist_len, dlist_alloc_len;
638 static void add_delete_entry(struct file_struct *file)
640 if (dlist_len == dlist_alloc_len) {
641 dlist_alloc_len += 1024;
643 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
645 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
647 if (!delete_list) out_of_memory("add_delete_entry");
650 delete_list[dlist_len].dev = file->dev;
651 delete_list[dlist_len].inode = file->inode;
655 fprintf(FINFO,"added %s to delete list\n", f_name(file));
658 /* yuck! This function wouldn't have been necessary if I had the sorting
659 algorithm right. Unfortunately fixing the sorting algorithm would introduce
660 a backward incompatibility as file list indexes are sent over the link.
662 static int delete_already_done(struct file_list *flist,int j)
667 if (link_stat(f_name(flist->files[j]), &st)) return 1;
669 for (i=0;i<dlist_len;i++) {
670 if (st.st_ino == delete_list[i].inode &&
671 st.st_dev == delete_list[i].dev)
679 /* this deletes any files on the receiving side that are not present
680 on the sending side. For version 1.6.4 I have changed the behaviour
681 to match more closely what most people seem to expect of this option */
682 static void delete_files(struct file_list *flist)
684 struct file_list *local_file_list;
692 fprintf(FINFO,"IO error encountered - skipping file deletion\n");
696 for (j=0;j<flist->count;j++) {
697 if (!S_ISDIR(flist->files[j]->mode) ||
698 !(flist->files[j]->flags & FLAG_DELETE)) continue;
700 if (delete_already_done(flist, j)) continue;
702 name = strdup(f_name(flist->files[j]));
704 if (!(local_file_list = send_file_list(-1,1,&name))) {
710 fprintf(FINFO,"deleting in %s\n", name);
712 for (i=local_file_list->count-1;i>=0;i--) {
713 if (!local_file_list->files[i]->basename) continue;
714 if (S_ISDIR(local_file_list->files[i]->mode))
715 add_delete_entry(local_file_list->files[i]);
716 if (-1 == flist_find(flist,local_file_list->files[i])) {
717 delete_one(local_file_list->files[i]);
720 flist_free(local_file_list);
725 static char *cleanup_fname;
727 void exit_cleanup(int code)
730 do_unlink(cleanup_fname);
731 signal(SIGUSR1, SIG_IGN);
744 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
749 char fnametmp[MAXPATHLEN];
750 struct map_struct *buf;
752 struct file_struct *file;
757 fprintf(FINFO,"recv_files(%d) starting\n",flist->count);
760 if (recurse && delete_mode && !local_name && flist->count>0) {
768 if (phase==0 && remote_version >= 13) {
770 csum_length = SUM_LENGTH;
772 fprintf(FINFO,"recv_files phase=%d\n",phase);
780 file = flist->files[i];
781 fname = f_name(file);
787 if (!am_server && verbose)
788 printf("%s\n",fname);
793 fprintf(FINFO,"recv_files(%s)\n",fname);
796 fd1 = open(fname,O_RDONLY);
798 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
799 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
800 receive_data(f_in,NULL,-1,NULL);
805 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
806 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
807 receive_data(f_in,NULL,-1,NULL);
812 if (fd1 != -1 && st.st_size > 0) {
813 buf = map_file(fd1,st.st_size);
815 fprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
821 if (strlen(fname) > (MAXPATHLEN-8)) {
822 fprintf(FERROR,"filename too long\n");
823 if (buf) unmap_file(buf);
829 f = strrchr(fname,'/');
834 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
836 sprintf(fnametmp,"%s.XXXXXX",fname);
838 if (NULL == do_mktemp(fnametmp)) {
839 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
840 receive_data(f_in,buf,-1,NULL);
841 if (buf) unmap_file(buf);
845 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
846 if (fd2 == -1 && relative_paths && errno == ENOENT &&
847 create_directory_path(fnametmp) == 0) {
848 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
851 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
852 receive_data(f_in,buf,-1,NULL);
853 if (buf) unmap_file(buf);
858 cleanup_fname = fnametmp;
860 if (!am_server && verbose)
861 printf("%s\n",fname);
864 recv_ok = receive_data(f_in,buf,fd2,fname);
866 if (buf) unmap_file(buf);
873 fprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
876 char fnamebak[MAXPATHLEN];
877 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
878 fprintf(FERROR,"backup filename too long\n");
881 sprintf(fnamebak,"%s%s",fname,backup_suffix);
882 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
883 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
888 /* move tmp file over real file */
889 if (do_rename(fnametmp,fname) != 0) {
890 if (errno == EXDEV) {
891 /* rename failed on cross-filesystem link.
892 Copy the file instead. */
893 if (copy_file(fnametmp,fname, file->mode)) {
894 fprintf(FERROR,"copy %s -> %s : %s\n",
895 fnametmp,fname,strerror(errno));
897 set_perms(fname,file,NULL,0);
901 fprintf(FERROR,"rename %s -> %s : %s\n",
902 fnametmp,fname,strerror(errno));
906 set_perms(fname,file,NULL,0);
909 cleanup_fname = NULL;
913 if (csum_length == SUM_LENGTH) {
914 fprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
918 fprintf(FINFO,"redoing %s(%d)\n",fname,i);
924 if (preserve_hard_links)
925 do_hard_links(flist);
927 /* now we need to fix any directory permissions that were
928 modified during the transfer */
929 for (i = 0; i < flist->count; i++) {
930 struct file_struct *file = flist->files[i];
931 if (!file->basename || !S_ISDIR(file->mode)) continue;
932 recv_generator(f_name(file),flist,i,-1);
936 fprintf(FINFO,"recv_files finished\n");
943 void send_files(struct file_list *flist,int f_out,int f_in)
946 struct sum_struct *s;
947 struct map_struct *buf;
949 char fname[MAXPATHLEN];
951 struct file_struct *file;
956 fprintf(FINFO,"send_files starting\n");
958 setup_nonblocking(f_in,f_out);
963 if (phase==0 && remote_version >= 13) {
965 csum_length = SUM_LENGTH;
969 fprintf(FINFO,"send_files phase=%d\n",phase);
975 file = flist->files[i];
979 strncpy(fname,file->basedir,MAXPATHLEN-1);
980 fname[MAXPATHLEN-1] = 0;
981 if (strlen(fname) == MAXPATHLEN-1) {
983 fprintf(FERROR, "send_files failed on long-named directory %s\n",
988 offset = strlen(file->basedir)+1;
990 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
993 fprintf(FINFO,"send_files(%d,%s)\n",i,fname);
996 if (!am_server && verbose)
997 printf("%s\n",fname);
1002 s = receive_sums(f_in);
1005 fprintf(FERROR,"receive_sums failed\n");
1009 fd = open(fname,O_RDONLY);
1012 fprintf(FERROR,"send_files failed to open %s: %s\n",
1013 fname,strerror(errno));
1018 /* map the local file */
1019 if (do_fstat(fd,&st) != 0) {
1021 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1027 if (st.st_size > 0) {
1028 buf = map_file(fd,st.st_size);
1034 fprintf(FINFO,"send_files mapped %s of size %d\n",
1035 fname,(int)st.st_size);
1039 write_int(f_out,s->count);
1040 write_int(f_out,s->n);
1041 write_int(f_out,s->remainder);
1044 fprintf(FINFO,"calling match_sums %s\n",fname);
1046 if (!am_server && verbose)
1047 printf("%s\n",fname+offset);
1049 match_sums(f_out,s,buf,st.st_size);
1052 if (buf) unmap_file(buf);
1058 fprintf(FINFO,"sender finished %s\n",fname);
1062 fprintf(FINFO,"send files finished\n");
1066 write_int(f_out,-1);
1072 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1078 fprintf(FINFO,"generator starting pid=%d count=%d\n",
1079 (int)getpid(),flist->count);
1081 for (i = 0; i < flist->count; i++) {
1082 struct file_struct *file = flist->files[i];
1083 mode_t saved_mode = file->mode;
1084 if (!file->basename) continue;
1086 /* we need to ensure that any directories we create have writeable
1087 permissions initially so that we can create the files within
1088 them. This is then fixed after the files are transferred */
1089 if (!am_root && S_ISDIR(file->mode)) {
1090 file->mode |= S_IWUSR; /* user write */
1093 recv_generator(local_name?local_name:f_name(file),
1096 file->mode = saved_mode;
1100 csum_length = SUM_LENGTH;
1104 fprintf(FINFO,"generate_files phase=%d\n",phase);
1109 /* we expect to just sit around now, so don't exit on a timeout. If we
1110 really get a timeout then the other process should exit */
1113 if (remote_version >= 13) {
1114 /* in newer versions of the protocol the files can cycle through
1115 the system more than once to catch initial checksum errors */
1116 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1117 struct file_struct *file = flist->files[i];
1118 recv_generator(local_name?local_name:f_name(file),
1124 fprintf(FINFO,"generate_files phase=%d\n",phase);
1132 fprintf(FINFO,"generator wrote %ld\n",(long)write_total());