2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 rprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 rprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 rprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 strlcpy(buf, fname, (MAXPATHLEN-strlen(dname))-2);
117 buf[MAXPATHLEN-1] = 0;
119 rprintf(FINFO,"deleting %s\n", buf);
120 if (delete_file(buf) != 0) {
128 if (do_rmdir(fname) != 0) {
129 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
137 send a sums struct down a fd
139 static void send_sums(struct sum_struct *s,int f_out)
143 /* tell the other guy how many we are going to be doing and how many
144 bytes there are in the last chunk */
145 write_int(f_out,s?s->count:0);
146 write_int(f_out,s?s->n:block_size);
147 write_int(f_out,s?s->remainder:0);
149 for (i=0;i<s->count;i++) {
150 write_int(f_out,s->sums[i].sum1);
151 write_buf(f_out,s->sums[i].sum2,csum_length);
158 generate a stream of signatures/checksums that describe a buffer
160 generate approximately one checksum every n bytes
162 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
165 struct sum_struct *s;
168 int remainder = (len%block_len);
171 count = (len+(block_len-1))/block_len;
173 s = (struct sum_struct *)malloc(sizeof(*s));
174 if (!s) out_of_memory("generate_sums");
177 s->remainder = remainder;
187 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
188 s->count,s->remainder,s->n,(int)s->flength);
190 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
191 if (!s->sums) out_of_memory("generate_sums");
193 for (i=0;i<count;i++) {
195 char *map = map_ptr(buf,offset,n1);
197 s->sums[i].sum1 = get_checksum1(map,n1);
198 get_checksum2(map,n1,s->sums[i].sum2);
200 s->sums[i].offset = offset;
205 rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
206 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
217 receive the checksums for a buffer
219 static struct sum_struct *receive_sums(int f)
221 struct sum_struct *s;
225 s = (struct sum_struct *)malloc(sizeof(*s));
226 if (!s) out_of_memory("receive_sums");
228 s->count = read_int(f);
230 s->remainder = read_int(f);
234 rprintf(FINFO,"count=%d n=%d rem=%d\n",
235 s->count,s->n,s->remainder);
240 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
241 if (!s->sums) out_of_memory("receive_sums");
243 for (i=0;i<s->count;i++) {
244 s->sums[i].sum1 = read_int(f);
245 read_buf(f,s->sums[i].sum2,csum_length);
247 s->sums[i].offset = offset;
250 if (i == s->count-1 && s->remainder != 0) {
251 s->sums[i].len = s->remainder;
253 s->sums[i].len = s->n;
255 offset += s->sums[i].len;
258 rprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
259 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
268 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
274 if (dry_run) return 0;
277 if (link_stat(fname,&st2) != 0) {
278 rprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
284 if (preserve_times && !S_ISLNK(st->st_mode) &&
285 st->st_mtime != file->modtime) {
287 if (set_modtime(fname,file->modtime) != 0) {
288 rprintf(FERROR,"failed to set times on %s : %s\n",
289 fname,strerror(errno));
295 if (preserve_perms && !S_ISLNK(st->st_mode) &&
296 st->st_mode != file->mode) {
298 if (do_chmod(fname,file->mode) != 0) {
299 rprintf(FERROR,"failed to set permissions on %s : %s\n",
300 fname,strerror(errno));
306 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
307 (preserve_gid && st->st_gid != file->gid)) {
309 (am_root&&preserve_uid)?file->uid:-1,
310 preserve_gid?file->gid:-1) != 0) {
311 if (preserve_uid && st->st_uid != file->uid)
313 if (verbose>1 || preserve_uid)
314 rprintf(FERROR,"chown %s : %s\n",
315 fname,strerror(errno));
321 if (verbose > 1 && report) {
323 rprintf(FINFO,"%s\n",fname);
325 rprintf(FINFO,"%s is uptodate\n",fname);
331 /* choose whether to skip a particular file */
332 static int skip_file(char *fname,
333 struct file_struct *file, STRUCT_STAT *st)
335 if (st->st_size != file->length) {
339 /* if always checksum is set then we use the checksum instead
340 of the file time to determine whether to sync */
341 if (always_checksum && S_ISREG(st->st_mode)) {
342 char sum[MD4_SUM_LENGTH];
343 file_checksum(fname,sum,st->st_size);
344 return (memcmp(sum,file->sum,csum_length) == 0);
351 return (st->st_mtime == file->modtime);
355 /* use a larger block size for really big files */
356 int adapt_block_size(struct file_struct *file, int bsize)
358 int ret = file->length / (10000); /* rough heuristic */
359 ret = ret & ~15; /* multiple of 16 */
360 if (ret < bsize) ret = bsize;
361 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
365 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
369 struct map_struct *buf;
370 struct sum_struct *s;
372 struct file_struct *file = flist->files[i];
375 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
377 statret = link_stat(fname,&st);
379 if (S_ISDIR(file->mode)) {
381 if (statret == 0 && !S_ISDIR(st.st_mode)) {
382 if (do_unlink(fname) != 0) {
383 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
388 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
389 if (!(relative_paths && errno==ENOENT &&
390 create_directory_path(fname)==0 &&
391 do_mkdir(fname,file->mode)==0)) {
392 rprintf(FERROR,"mkdir %s : %s (2)\n",
393 fname,strerror(errno));
396 if (set_perms(fname,file,NULL,0) && verbose)
397 rprintf(FINFO,"%s/\n",fname);
401 if (preserve_links && S_ISLNK(file->mode)) {
403 char lnk[MAXPATHLEN];
406 l = readlink(fname,lnk,MAXPATHLEN-1);
409 if (strcmp(lnk,file->link) == 0) {
410 set_perms(fname,file,&st,1);
416 if (do_symlink(file->link,fname) != 0) {
417 rprintf(FERROR,"link %s -> %s : %s\n",
418 fname,file->link,strerror(errno));
420 set_perms(fname,file,NULL,0);
422 rprintf(FINFO,"%s -> %s\n",
430 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
432 st.st_mode != file->mode ||
433 st.st_rdev != file->rdev) {
436 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
437 fname,(int)file->mode,(int)file->rdev);
438 if (do_mknod(fname,file->mode,file->rdev) != 0) {
439 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
441 set_perms(fname,file,NULL,0);
443 rprintf(FINFO,"%s\n",fname);
446 set_perms(fname,file,&st,1);
452 if (preserve_hard_links && check_hard_link(file)) {
454 rprintf(FINFO,"%s is a hard link\n",f_name(file));
458 if (!S_ISREG(file->mode)) {
459 rprintf(FINFO,"skipping non-regular file %s\n",fname);
464 if (errno == ENOENT) {
466 if (!dry_run) send_sums(NULL,f_out);
469 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
474 if (!S_ISREG(st.st_mode)) {
475 if (delete_file(fname) != 0) {
479 /* now pretend the file didn't exist */
481 if (!dry_run) send_sums(NULL,f_out);
485 if (update_only && st.st_mtime > file->modtime) {
487 rprintf(FINFO,"%s is newer\n",fname);
491 if (skip_file(fname, file, &st)) {
492 set_perms(fname,file,&st,1);
503 send_sums(NULL,f_out);
508 fd = open(fname,O_RDONLY);
511 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
512 rprintf(FERROR,"skipping %s\n",fname);
516 if (st.st_size > 0) {
517 buf = map_file(fd,st.st_size);
523 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
525 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
528 rprintf(FINFO,"sending sums for %d\n",i);
535 if (buf) unmap_file(buf);
542 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
544 int i,n,remainder,len,count;
548 static char file_sum1[MD4_SUM_LENGTH];
549 static char file_sum2[MD4_SUM_LENGTH];
552 count = read_int(f_in);
554 remainder = read_int(f_in);
558 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
561 rprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
565 if (fd != -1 && write_file(fd,data,i) != i) {
566 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
574 if (i == count-1 && remainder != 0)
578 rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
579 i,len,(int)offset2,(int)offset);
581 map = map_ptr(buf,offset2,len);
586 if (fd != -1 && write_file(fd,map,len) != len) {
587 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
594 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
595 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
601 if (remote_version >= 14) {
602 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
604 rprintf(FINFO,"got file_sum\n");
605 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
612 static void delete_one(struct file_struct *f)
614 if (!S_ISDIR(f->mode)) {
615 if (do_unlink(f_name(f)) != 0) {
616 rprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
617 } else if (verbose) {
618 rprintf(FINFO,"deleting %s\n",f_name(f));
621 if (do_rmdir(f_name(f)) != 0) {
622 if (errno != ENOTEMPTY && errno != EEXIST)
623 rprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
624 } else if (verbose) {
625 rprintf(FINFO,"deleting directory %s\n",f_name(f));
632 static struct delete_list {
636 static int dlist_len, dlist_alloc_len;
638 static void add_delete_entry(struct file_struct *file)
640 if (dlist_len == dlist_alloc_len) {
641 dlist_alloc_len += 1024;
643 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
645 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
647 if (!delete_list) out_of_memory("add_delete_entry");
650 delete_list[dlist_len].dev = file->dev;
651 delete_list[dlist_len].inode = file->inode;
655 rprintf(FINFO,"added %s to delete list\n", f_name(file));
658 /* yuck! This function wouldn't have been necessary if I had the sorting
659 algorithm right. Unfortunately fixing the sorting algorithm would introduce
660 a backward incompatibility as file list indexes are sent over the link.
662 static int delete_already_done(struct file_list *flist,int j)
667 if (link_stat(f_name(flist->files[j]), &st)) return 1;
669 for (i=0;i<dlist_len;i++) {
670 if (st.st_ino == delete_list[i].inode &&
671 st.st_dev == delete_list[i].dev)
679 /* this deletes any files on the receiving side that are not present
680 on the sending side. For version 1.6.4 I have changed the behaviour
681 to match more closely what most people seem to expect of this option */
682 static void delete_files(struct file_list *flist)
684 struct file_list *local_file_list;
692 rprintf(FINFO,"IO error encountered - skipping file deletion\n");
696 for (j=0;j<flist->count;j++) {
697 if (!S_ISDIR(flist->files[j]->mode) ||
698 !(flist->files[j]->flags & FLAG_DELETE)) continue;
700 if (delete_already_done(flist, j)) continue;
702 name = strdup(f_name(flist->files[j]));
704 if (!(local_file_list = send_file_list(-1,1,&name))) {
710 rprintf(FINFO,"deleting in %s\n", name);
712 for (i=local_file_list->count-1;i>=0;i--) {
713 if (!local_file_list->files[i]->basename) continue;
714 if (S_ISDIR(local_file_list->files[i]->mode))
715 add_delete_entry(local_file_list->files[i]);
716 if (-1 == flist_find(flist,local_file_list->files[i])) {
717 delete_one(local_file_list->files[i]);
720 flist_free(local_file_list);
725 static char *cleanup_fname;
727 void exit_cleanup(int code)
730 do_unlink(cleanup_fname);
731 signal(SIGUSR1, SIG_IGN);
744 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
749 char fnametmp[MAXPATHLEN];
750 struct map_struct *buf;
752 struct file_struct *file;
757 rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
760 if (recurse && delete_mode && !local_name && flist->count>0) {
768 if (phase==0 && remote_version >= 13) {
770 csum_length = SUM_LENGTH;
772 rprintf(FINFO,"recv_files phase=%d\n",phase);
780 file = flist->files[i];
781 fname = f_name(file);
787 if (!am_server && verbose)
788 printf("%s\n",fname);
793 rprintf(FINFO,"recv_files(%s)\n",fname);
796 fd1 = open(fname,O_RDONLY);
798 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
799 rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
800 receive_data(f_in,NULL,-1,NULL);
805 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
806 rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
807 receive_data(f_in,NULL,-1,NULL);
812 if (fd1 != -1 && st.st_size > 0) {
813 buf = map_file(fd1,st.st_size);
815 rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
823 f = strrchr(fname,'/');
828 if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) {
829 rprintf(FERROR,"filename too long\n");
830 if (buf) unmap_file(buf);
834 sprintf(fnametmp,"%s/.%s.XXXXXX",tmpdir,f);
836 if (strlen(fname)+9 > MAXPATHLEN) {
837 rprintf(FERROR,"filename too long\n");
838 if (buf) unmap_file(buf);
843 sprintf(fnametmp,".%s.XXXXXX",fname);
845 if (NULL == do_mktemp(fnametmp)) {
846 rprintf(FERROR,"mktemp %s failed\n",fnametmp);
847 receive_data(f_in,buf,-1,NULL);
848 if (buf) unmap_file(buf);
852 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
853 if (fd2 == -1 && relative_paths && errno == ENOENT &&
854 create_directory_path(fnametmp) == 0) {
855 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
858 rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
859 receive_data(f_in,buf,-1,NULL);
860 if (buf) unmap_file(buf);
865 cleanup_fname = fnametmp;
867 if (!am_server && verbose)
868 printf("%s\n",fname);
871 recv_ok = receive_data(f_in,buf,fd2,fname);
873 if (buf) unmap_file(buf);
880 rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
883 char fnamebak[MAXPATHLEN];
884 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
885 rprintf(FERROR,"backup filename too long\n");
888 sprintf(fnamebak,"%s%s",fname,backup_suffix);
889 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
890 rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
895 /* move tmp file over real file */
896 if (do_rename(fnametmp,fname) != 0) {
897 if (errno == EXDEV) {
898 /* rename failed on cross-filesystem link.
899 Copy the file instead. */
900 if (copy_file(fnametmp,fname, file->mode)) {
901 rprintf(FERROR,"copy %s -> %s : %s\n",
902 fnametmp,fname,strerror(errno));
904 set_perms(fname,file,NULL,0);
908 rprintf(FERROR,"rename %s -> %s : %s\n",
909 fnametmp,fname,strerror(errno));
913 set_perms(fname,file,NULL,0);
916 cleanup_fname = NULL;
920 if (csum_length == SUM_LENGTH) {
921 rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
925 rprintf(FINFO,"redoing %s(%d)\n",fname,i);
931 if (preserve_hard_links)
932 do_hard_links(flist);
934 /* now we need to fix any directory permissions that were
935 modified during the transfer */
936 for (i = 0; i < flist->count; i++) {
937 struct file_struct *file = flist->files[i];
938 if (!file->basename || !S_ISDIR(file->mode)) continue;
939 recv_generator(f_name(file),flist,i,-1);
943 rprintf(FINFO,"recv_files finished\n");
950 void send_files(struct file_list *flist,int f_out,int f_in)
953 struct sum_struct *s;
954 struct map_struct *buf;
956 char fname[MAXPATHLEN];
958 struct file_struct *file;
963 rprintf(FINFO,"send_files starting\n");
965 setup_nonblocking(f_in,f_out);
970 if (phase==0 && remote_version >= 13) {
972 csum_length = SUM_LENGTH;
976 rprintf(FINFO,"send_files phase=%d\n",phase);
982 file = flist->files[i];
986 strlcpy(fname,file->basedir,MAXPATHLEN-1);
987 if (strlen(fname) == MAXPATHLEN-1) {
989 rprintf(FERROR, "send_files failed on long-named directory %s\n",
994 offset = strlen(file->basedir)+1;
996 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
999 rprintf(FINFO,"send_files(%d,%s)\n",i,fname);
1002 if (!am_server && verbose)
1003 printf("%s\n",fname);
1008 s = receive_sums(f_in);
1011 rprintf(FERROR,"receive_sums failed\n");
1015 fd = open(fname,O_RDONLY);
1018 rprintf(FERROR,"send_files failed to open %s: %s\n",
1019 fname,strerror(errno));
1024 /* map the local file */
1025 if (do_fstat(fd,&st) != 0) {
1027 rprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1033 if (st.st_size > 0) {
1034 buf = map_file(fd,st.st_size);
1040 rprintf(FINFO,"send_files mapped %s of size %d\n",
1041 fname,(int)st.st_size);
1045 write_int(f_out,s->count);
1046 write_int(f_out,s->n);
1047 write_int(f_out,s->remainder);
1050 rprintf(FINFO,"calling match_sums %s\n",fname);
1052 if (!am_server && verbose)
1053 printf("%s\n",fname+offset);
1055 match_sums(f_out,s,buf,st.st_size);
1058 if (buf) unmap_file(buf);
1064 rprintf(FINFO,"sender finished %s\n",fname);
1068 rprintf(FINFO,"send files finished\n");
1072 write_int(f_out,-1);
1078 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1084 rprintf(FINFO,"generator starting pid=%d count=%d\n",
1085 (int)getpid(),flist->count);
1087 for (i = 0; i < flist->count; i++) {
1088 struct file_struct *file = flist->files[i];
1089 mode_t saved_mode = file->mode;
1090 if (!file->basename) continue;
1092 /* we need to ensure that any directories we create have writeable
1093 permissions initially so that we can create the files within
1094 them. This is then fixed after the files are transferred */
1095 if (!am_root && S_ISDIR(file->mode)) {
1096 file->mode |= S_IWUSR; /* user write */
1099 recv_generator(local_name?local_name:f_name(file),
1102 file->mode = saved_mode;
1106 csum_length = SUM_LENGTH;
1110 rprintf(FINFO,"generate_files phase=%d\n",phase);
1115 /* we expect to just sit around now, so don't exit on a timeout. If we
1116 really get a timeout then the other process should exit */
1119 if (remote_version >= 13) {
1120 /* in newer versions of the protocol the files can cycle through
1121 the system more than once to catch initial checksum errors */
1122 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1123 struct file_struct *file = flist->files[i];
1124 recv_generator(local_name?local_name:f_name(file),
1130 rprintf(FINFO,"generate_files phase=%d\n",phase);
1138 rprintf(FINFO,"generator wrote %ld\n",(long)write_total());