2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = do_lstat(fname, &st);
83 ret = do_stat(fname, &st);
86 rprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 rprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 rprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 char *dname = d_name(di);
111 if (strcmp(dname,".")==0 ||
112 strcmp(dname,"..")==0)
114 strlcpy(buf, fname, (MAXPATHLEN-strlen(dname))-2);
117 buf[MAXPATHLEN-1] = 0;
119 rprintf(FINFO,"deleting %s\n", buf);
120 if (delete_file(buf) != 0) {
128 if (do_rmdir(fname) != 0) {
129 rprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
137 send a sums struct down a fd
139 static void send_sums(struct sum_struct *s,int f_out)
143 /* tell the other guy how many we are going to be doing and how many
144 bytes there are in the last chunk */
145 write_int(f_out,s?s->count:0);
146 write_int(f_out,s?s->n:block_size);
147 write_int(f_out,s?s->remainder:0);
149 for (i=0;i<s->count;i++) {
150 write_int(f_out,s->sums[i].sum1);
151 write_buf(f_out,s->sums[i].sum2,csum_length);
158 generate a stream of signatures/checksums that describe a buffer
160 generate approximately one checksum every n bytes
162 static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
165 struct sum_struct *s;
168 int remainder = (len%block_len);
171 count = (len+(block_len-1))/block_len;
173 s = (struct sum_struct *)malloc(sizeof(*s));
174 if (!s) out_of_memory("generate_sums");
177 s->remainder = remainder;
187 rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
188 s->count,s->remainder,s->n,(int)s->flength);
190 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
191 if (!s->sums) out_of_memory("generate_sums");
193 for (i=0;i<count;i++) {
195 char *map = map_ptr(buf,offset,n1);
197 s->sums[i].sum1 = get_checksum1(map,n1);
198 get_checksum2(map,n1,s->sums[i].sum2);
200 s->sums[i].offset = offset;
205 rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
206 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
217 receive the checksums for a buffer
219 static struct sum_struct *receive_sums(int f)
221 struct sum_struct *s;
225 s = (struct sum_struct *)malloc(sizeof(*s));
226 if (!s) out_of_memory("receive_sums");
228 s->count = read_int(f);
230 s->remainder = read_int(f);
234 rprintf(FINFO,"count=%d n=%d rem=%d\n",
235 s->count,s->n,s->remainder);
240 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
241 if (!s->sums) out_of_memory("receive_sums");
243 for (i=0;i<s->count;i++) {
244 s->sums[i].sum1 = read_int(f);
245 read_buf(f,s->sums[i].sum2,csum_length);
247 s->sums[i].offset = offset;
250 if (i == s->count-1 && s->remainder != 0) {
251 s->sums[i].len = s->remainder;
253 s->sums[i].len = s->n;
255 offset += s->sums[i].len;
258 rprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
259 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
268 static int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st,
274 if (dry_run) return 0;
277 if (link_stat(fname,&st2) != 0) {
278 rprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
284 if (preserve_times && !S_ISLNK(st->st_mode) &&
285 st->st_mtime != file->modtime) {
287 if (set_modtime(fname,file->modtime) != 0) {
288 rprintf(FERROR,"failed to set times on %s : %s\n",
289 fname,strerror(errno));
295 if (preserve_perms && !S_ISLNK(st->st_mode) &&
296 st->st_mode != file->mode) {
298 if (do_chmod(fname,file->mode) != 0) {
299 rprintf(FERROR,"failed to set permissions on %s : %s\n",
300 fname,strerror(errno));
306 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
307 (preserve_gid && st->st_gid != file->gid)) {
309 (am_root&&preserve_uid)?file->uid:-1,
310 preserve_gid?file->gid:-1) != 0) {
311 if (preserve_uid && st->st_uid != file->uid)
313 if (verbose>1 || preserve_uid)
314 rprintf(FERROR,"chown %s : %s\n",
315 fname,strerror(errno));
321 if (verbose > 1 && report) {
323 rprintf(FINFO,"%s\n",fname);
325 rprintf(FINFO,"%s is uptodate\n",fname);
331 /* choose whether to skip a particular file */
332 static int skip_file(char *fname,
333 struct file_struct *file, STRUCT_STAT *st)
335 if (st->st_size != file->length) {
339 /* if always checksum is set then we use the checksum instead
340 of the file time to determine whether to sync */
341 if (always_checksum && S_ISREG(st->st_mode)) {
342 char sum[MD4_SUM_LENGTH];
343 file_checksum(fname,sum,st->st_size);
344 return (memcmp(sum,file->sum,csum_length) == 0);
351 return (st->st_mtime == file->modtime);
355 /* use a larger block size for really big files */
356 int adapt_block_size(struct file_struct *file, int bsize)
358 int ret = file->length / (10000); /* rough heuristic */
359 ret = ret & ~15; /* multiple of 16 */
360 if (ret < bsize) ret = bsize;
361 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
365 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
369 struct map_struct *buf;
370 struct sum_struct *s;
372 struct file_struct *file = flist->files[i];
375 rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
377 statret = link_stat(fname,&st);
379 if (S_ISDIR(file->mode)) {
381 if (statret == 0 && !S_ISDIR(st.st_mode)) {
382 if (do_unlink(fname) != 0) {
383 rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
388 if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
389 if (!(relative_paths && errno==ENOENT &&
390 create_directory_path(fname)==0 &&
391 do_mkdir(fname,file->mode)==0)) {
392 rprintf(FERROR,"mkdir %s : %s (2)\n",
393 fname,strerror(errno));
396 if (set_perms(fname,file,NULL,0) && verbose)
397 rprintf(FINFO,"%s/\n",fname);
401 if (preserve_links && S_ISLNK(file->mode)) {
403 char lnk[MAXPATHLEN];
406 l = readlink(fname,lnk,MAXPATHLEN-1);
409 if (strcmp(lnk,file->link) == 0) {
410 set_perms(fname,file,&st,1);
416 if (do_symlink(file->link,fname) != 0) {
417 rprintf(FERROR,"link %s -> %s : %s\n",
418 fname,file->link,strerror(errno));
420 set_perms(fname,file,NULL,0);
422 rprintf(FINFO,"%s -> %s\n",
430 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
432 st.st_mode != file->mode ||
433 st.st_rdev != file->rdev) {
436 rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
437 fname,(int)file->mode,(int)file->rdev);
438 if (do_mknod(fname,file->mode,file->rdev) != 0) {
439 rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
441 set_perms(fname,file,NULL,0);
443 rprintf(FINFO,"%s\n",fname);
446 set_perms(fname,file,&st,1);
452 if (preserve_hard_links && check_hard_link(file)) {
454 rprintf(FINFO,"%s is a hard link\n",f_name(file));
458 if (!S_ISREG(file->mode)) {
459 rprintf(FINFO,"skipping non-regular file %s\n",fname);
464 if (errno == ENOENT) {
466 if (!dry_run) send_sums(NULL,f_out);
469 rprintf(FERROR,"recv_generator failed to open %s\n",fname);
474 if (!S_ISREG(st.st_mode)) {
475 if (delete_file(fname) != 0) {
479 /* now pretend the file didn't exist */
481 if (!dry_run) send_sums(NULL,f_out);
485 if (update_only && st.st_mtime > file->modtime) {
487 rprintf(FINFO,"%s is newer\n",fname);
491 if (skip_file(fname, file, &st)) {
492 set_perms(fname,file,&st,1);
503 send_sums(NULL,f_out);
508 fd = open(fname,O_RDONLY);
511 rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
512 rprintf(FERROR,"skipping %s\n",fname);
516 if (st.st_size > 0) {
517 buf = map_file(fd,st.st_size);
523 rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
525 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
528 rprintf(FINFO,"sending sums for %d\n",i);
535 if (buf) unmap_file(buf);
542 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
544 int i,n,remainder,len,count;
548 static char file_sum1[MD4_SUM_LENGTH];
549 static char file_sum2[MD4_SUM_LENGTH];
552 count = read_int(f_in);
554 remainder = read_int(f_in);
558 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
561 rprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
565 if (fd != -1 && write_file(fd,data,i) != i) {
566 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
574 if (i == count-1 && remainder != 0)
578 rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
579 i,len,(int)offset2,(int)offset);
581 map = map_ptr(buf,offset2,len);
586 if (fd != -1 && write_file(fd,map,len) != len) {
587 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
594 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
595 rprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
601 if (remote_version >= 14) {
602 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
604 rprintf(FINFO,"got file_sum\n");
605 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
612 static void delete_one(struct file_struct *f)
614 if (!S_ISDIR(f->mode)) {
615 if (do_unlink(f_name(f)) != 0) {
616 rprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
617 } else if (verbose) {
618 rprintf(FINFO,"deleting %s\n",f_name(f));
621 if (do_rmdir(f_name(f)) != 0) {
622 if (errno != ENOTEMPTY && errno != EEXIST)
623 rprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
624 } else if (verbose) {
625 rprintf(FINFO,"deleting directory %s\n",f_name(f));
632 static struct delete_list {
636 static int dlist_len, dlist_alloc_len;
638 static void add_delete_entry(struct file_struct *file)
640 if (dlist_len == dlist_alloc_len) {
641 dlist_alloc_len += 1024;
643 delete_list = (struct delete_list *)malloc(sizeof(delete_list[0])*dlist_alloc_len);
645 delete_list = (struct delete_list *)realloc(delete_list, sizeof(delete_list[0])*dlist_alloc_len);
647 if (!delete_list) out_of_memory("add_delete_entry");
650 delete_list[dlist_len].dev = file->dev;
651 delete_list[dlist_len].inode = file->inode;
655 rprintf(FINFO,"added %s to delete list\n", f_name(file));
658 /* yuck! This function wouldn't have been necessary if I had the sorting
659 algorithm right. Unfortunately fixing the sorting algorithm would introduce
660 a backward incompatibility as file list indexes are sent over the link.
662 static int delete_already_done(struct file_list *flist,int j)
667 if (link_stat(f_name(flist->files[j]), &st)) return 1;
669 for (i=0;i<dlist_len;i++) {
670 if (st.st_ino == delete_list[i].inode &&
671 st.st_dev == delete_list[i].dev)
679 /* this deletes any files on the receiving side that are not present
680 on the sending side. For version 1.6.4 I have changed the behaviour
681 to match more closely what most people seem to expect of this option */
682 static void delete_files(struct file_list *flist)
684 struct file_list *local_file_list;
692 rprintf(FINFO,"IO error encountered - skipping file deletion\n");
696 for (j=0;j<flist->count;j++) {
697 if (!S_ISDIR(flist->files[j]->mode) ||
698 !(flist->files[j]->flags & FLAG_DELETE)) continue;
700 if (delete_already_done(flist, j)) continue;
702 name = strdup(f_name(flist->files[j]));
704 if (!(local_file_list = send_file_list(-1,1,&name))) {
710 rprintf(FINFO,"deleting in %s\n", name);
712 for (i=local_file_list->count-1;i>=0;i--) {
713 if (!local_file_list->files[i]->basename) continue;
714 if (S_ISDIR(local_file_list->files[i]->mode))
715 add_delete_entry(local_file_list->files[i]);
716 if (-1 == flist_find(flist,local_file_list->files[i])) {
717 delete_one(local_file_list->files[i]);
720 flist_free(local_file_list);
725 static char *cleanup_fname;
727 void exit_cleanup(int code)
730 do_unlink(cleanup_fname);
731 signal(SIGUSR1, SIG_IGN);
744 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
749 char fnametmp[MAXPATHLEN];
750 struct map_struct *buf;
752 struct file_struct *file;
757 rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
760 if (recurse && delete_mode && !local_name && flist->count>0) {
768 if (phase==0 && remote_version >= 13) {
770 csum_length = SUM_LENGTH;
772 rprintf(FINFO,"recv_files phase=%d\n",phase);
780 file = flist->files[i];
781 fname = f_name(file);
787 if (!am_server && verbose)
788 printf("%s\n",fname);
793 rprintf(FINFO,"recv_files(%s)\n",fname);
796 fd1 = open(fname,O_RDONLY);
798 if (fd1 != -1 && do_fstat(fd1,&st) != 0) {
799 rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
800 receive_data(f_in,NULL,-1,NULL);
805 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
806 rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
807 receive_data(f_in,NULL,-1,NULL);
812 if (fd1 != -1 && st.st_size > 0) {
813 buf = map_file(fd1,st.st_size);
815 rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
823 f = strrchr(fname,'/');
828 if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) {
829 rprintf(FERROR,"filename too long\n");
830 if (buf) unmap_file(buf);
834 sprintf(fnametmp,"%s/.%s.XXXXXX",tmpdir,f);
836 char *f = strrchr(fname,'/');
838 if (strlen(fname)+9 > MAXPATHLEN) {
839 rprintf(FERROR,"filename too long\n");
840 if (buf) unmap_file(buf);
847 sprintf(fnametmp,"%s/.%s.XXXXXX",fname,f+1);
850 sprintf(fnametmp,".%s.XXXXXX",fname);
853 if (NULL == do_mktemp(fnametmp)) {
854 rprintf(FERROR,"mktemp %s failed\n",fnametmp);
855 receive_data(f_in,buf,-1,NULL);
856 if (buf) unmap_file(buf);
860 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
861 if (fd2 == -1 && relative_paths && errno == ENOENT &&
862 create_directory_path(fnametmp) == 0) {
863 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
866 rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
867 receive_data(f_in,buf,-1,NULL);
868 if (buf) unmap_file(buf);
873 cleanup_fname = fnametmp;
875 if (!am_server && verbose)
876 printf("%s\n",fname);
879 recv_ok = receive_data(f_in,buf,fd2,fname);
881 if (buf) unmap_file(buf);
888 rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
891 char fnamebak[MAXPATHLEN];
892 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
893 rprintf(FERROR,"backup filename too long\n");
896 sprintf(fnamebak,"%s%s",fname,backup_suffix);
897 if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) {
898 rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
903 /* move tmp file over real file */
904 if (do_rename(fnametmp,fname) != 0) {
905 if (errno == EXDEV) {
906 /* rename failed on cross-filesystem link.
907 Copy the file instead. */
908 if (copy_file(fnametmp,fname, file->mode)) {
909 rprintf(FERROR,"copy %s -> %s : %s\n",
910 fnametmp,fname,strerror(errno));
912 set_perms(fname,file,NULL,0);
916 rprintf(FERROR,"rename %s -> %s : %s\n",
917 fnametmp,fname,strerror(errno));
921 set_perms(fname,file,NULL,0);
924 cleanup_fname = NULL;
928 if (csum_length == SUM_LENGTH) {
929 rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
933 rprintf(FINFO,"redoing %s(%d)\n",fname,i);
939 if (preserve_hard_links)
940 do_hard_links(flist);
942 /* now we need to fix any directory permissions that were
943 modified during the transfer */
944 for (i = 0; i < flist->count; i++) {
945 struct file_struct *file = flist->files[i];
946 if (!file->basename || !S_ISDIR(file->mode)) continue;
947 recv_generator(f_name(file),flist,i,-1);
951 rprintf(FINFO,"recv_files finished\n");
958 void send_files(struct file_list *flist,int f_out,int f_in)
961 struct sum_struct *s;
962 struct map_struct *buf;
964 char fname[MAXPATHLEN];
966 struct file_struct *file;
971 rprintf(FINFO,"send_files starting\n");
973 setup_nonblocking(f_in,f_out);
978 if (phase==0 && remote_version >= 13) {
980 csum_length = SUM_LENGTH;
984 rprintf(FINFO,"send_files phase=%d\n",phase);
990 file = flist->files[i];
994 strlcpy(fname,file->basedir,MAXPATHLEN-1);
995 if (strlen(fname) == MAXPATHLEN-1) {
997 rprintf(FERROR, "send_files failed on long-named directory %s\n",
1002 offset = strlen(file->basedir)+1;
1004 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
1007 rprintf(FINFO,"send_files(%d,%s)\n",i,fname);
1010 if (!am_server && verbose)
1011 printf("%s\n",fname);
1016 s = receive_sums(f_in);
1019 rprintf(FERROR,"receive_sums failed\n");
1023 fd = open(fname,O_RDONLY);
1026 rprintf(FERROR,"send_files failed to open %s: %s\n",
1027 fname,strerror(errno));
1032 /* map the local file */
1033 if (do_fstat(fd,&st) != 0) {
1035 rprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1041 if (st.st_size > 0) {
1042 buf = map_file(fd,st.st_size);
1048 rprintf(FINFO,"send_files mapped %s of size %d\n",
1049 fname,(int)st.st_size);
1053 write_int(f_out,s->count);
1054 write_int(f_out,s->n);
1055 write_int(f_out,s->remainder);
1058 rprintf(FINFO,"calling match_sums %s\n",fname);
1060 if (!am_server && verbose)
1061 printf("%s\n",fname+offset);
1063 match_sums(f_out,s,buf,st.st_size);
1066 if (buf) unmap_file(buf);
1072 rprintf(FINFO,"sender finished %s\n",fname);
1076 rprintf(FINFO,"send files finished\n");
1080 write_int(f_out,-1);
1086 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1092 rprintf(FINFO,"generator starting pid=%d count=%d\n",
1093 (int)getpid(),flist->count);
1095 for (i = 0; i < flist->count; i++) {
1096 struct file_struct *file = flist->files[i];
1097 mode_t saved_mode = file->mode;
1098 if (!file->basename) continue;
1100 /* we need to ensure that any directories we create have writeable
1101 permissions initially so that we can create the files within
1102 them. This is then fixed after the files are transferred */
1103 if (!am_root && S_ISDIR(file->mode)) {
1104 file->mode |= S_IWUSR; /* user write */
1107 recv_generator(local_name?local_name:f_name(file),
1110 file->mode = saved_mode;
1114 csum_length = SUM_LENGTH;
1118 rprintf(FINFO,"generate_files phase=%d\n",phase);
1123 /* we expect to just sit around now, so don't exit on a timeout. If we
1124 really get a timeout then the other process should exit */
1127 if (remote_version >= 13) {
1128 /* in newer versions of the protocol the files can cycle through
1129 the system more than once to catch initial checksum errors */
1130 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1131 struct file_struct *file = flist->files[i];
1132 recv_generator(local_name?local_name:f_name(file),
1138 rprintf(FINFO,"generate_files phase=%d\n",phase);
1146 rprintf(FINFO,"generator wrote %ld\n",(long)write_total());