2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
52 extern int io_timeout;
58 static void free_sums(struct sum_struct *s)
60 if (s->sums) free(s->sums);
66 * delete a file or directory. If force_delet is set then delete
69 static int delete_file(char *fname)
74 extern int force_delete;
78 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
81 ret = lstat(fname, &st);
83 ret = stat(fname, &st);
86 fprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
90 if (!S_ISDIR(st.st_mode)) {
91 fprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
95 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
96 if (!force_delete || (errno != ENOTEMPTY && errno != EEXIST)) {
97 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
101 /* now we do a recsursive delete on the directory ... */
104 fprintf(FERROR,"opendir(%s): %s\n",
105 fname,strerror(errno));
109 for (di=readdir(d); di; di=readdir(d)) {
110 if (strcmp(di->d_name,".")==0 ||
111 strcmp(di->d_name,"..")==0)
113 strncpy(buf, fname, (MAXPATHLEN-strlen(di->d_name))-2);
115 strcat(buf, di->d_name);
116 buf[MAXPATHLEN-1] = 0;
118 fprintf(FINFO,"deleting %s\n", buf);
119 if (delete_file(buf) != 0) {
127 if (do_rmdir(fname) != 0) {
128 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
136 send a sums struct down a fd
138 static void send_sums(struct sum_struct *s,int f_out)
142 /* tell the other guy how many we are going to be doing and how many
143 bytes there are in the last chunk */
144 write_int(f_out,s?s->count:0);
145 write_int(f_out,s?s->n:block_size);
146 write_int(f_out,s?s->remainder:0);
148 for (i=0;i<s->count;i++) {
149 write_int(f_out,s->sums[i].sum1);
150 write_buf(f_out,s->sums[i].sum2,csum_length);
157 generate a stream of signatures/checksums that describe a buffer
159 generate approximately one checksum every n bytes
161 static struct sum_struct *generate_sums(struct map_struct *buf,off_t len,int n)
164 struct sum_struct *s;
167 int remainder = (len%block_len);
170 count = (len+(block_len-1))/block_len;
172 s = (struct sum_struct *)malloc(sizeof(*s));
173 if (!s) out_of_memory("generate_sums");
176 s->remainder = remainder;
186 fprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
187 s->count,s->remainder,s->n,(int)s->flength);
189 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
190 if (!s->sums) out_of_memory("generate_sums");
192 for (i=0;i<count;i++) {
194 char *map = map_ptr(buf,offset,n1);
196 s->sums[i].sum1 = get_checksum1(map,n1);
197 get_checksum2(map,n1,s->sums[i].sum2);
199 s->sums[i].offset = offset;
204 fprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
205 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
216 receive the checksums for a buffer
218 static struct sum_struct *receive_sums(int f)
220 struct sum_struct *s;
224 s = (struct sum_struct *)malloc(sizeof(*s));
225 if (!s) out_of_memory("receive_sums");
227 s->count = read_int(f);
229 s->remainder = read_int(f);
233 fprintf(FINFO,"count=%d n=%d rem=%d\n",
234 s->count,s->n,s->remainder);
239 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
240 if (!s->sums) out_of_memory("receive_sums");
242 for (i=0;i<s->count;i++) {
243 s->sums[i].sum1 = read_int(f);
244 read_buf(f,s->sums[i].sum2,csum_length);
246 s->sums[i].offset = offset;
249 if (i == s->count-1 && s->remainder != 0) {
250 s->sums[i].len = s->remainder;
252 s->sums[i].len = s->n;
254 offset += s->sums[i].len;
257 fprintf(FINFO,"chunk[%d] len=%d offset=%d sum1=%08x\n",
258 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
267 static int set_perms(char *fname,struct file_struct *file,struct stat *st,
273 if (dry_run) return 0;
276 if (link_stat(fname,&st2) != 0) {
277 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
283 if (preserve_times && !S_ISLNK(st->st_mode) &&
284 st->st_mtime != file->modtime) {
286 if (set_modtime(fname,file->modtime) != 0) {
287 fprintf(FERROR,"failed to set times on %s : %s\n",
288 fname,strerror(errno));
294 if (preserve_perms && !S_ISLNK(st->st_mode) &&
295 st->st_mode != file->mode) {
297 if (do_chmod(fname,file->mode) != 0) {
298 fprintf(FERROR,"failed to set permissions on %s : %s\n",
299 fname,strerror(errno));
305 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
306 (preserve_gid && st->st_gid != file->gid)) {
308 (am_root&&preserve_uid)?file->uid:-1,
309 preserve_gid?file->gid:-1) != 0) {
310 if (preserve_uid && st->st_uid != file->uid)
312 if (verbose>1 || preserve_uid)
313 fprintf(FERROR,"chown %s : %s\n",
314 fname,strerror(errno));
320 if (verbose > 1 && report) {
322 fprintf(FINFO,"%s\n",fname);
324 fprintf(FINFO,"%s is uptodate\n",fname);
330 /* choose whether to skip a particular file */
331 static int skip_file(char *fname,
332 struct file_struct *file, struct stat *st)
334 if (st->st_size != file->length) {
338 /* if always checksum is set then we use the checksum instead
339 of the file time to determine whether to sync */
340 if (always_checksum && S_ISREG(st->st_mode)) {
341 char sum[MD4_SUM_LENGTH];
342 file_checksum(fname,sum,st->st_size);
343 return (memcmp(sum,file->sum,csum_length) == 0);
350 return (st->st_mtime == file->modtime);
354 /* use a larger block size for really big files */
355 int adapt_block_size(struct file_struct *file, int bsize)
357 int ret = file->length / (10000); /* rough heuristic */
358 ret = ret & ~15; /* multiple of 16 */
359 if (ret < bsize) ret = bsize;
360 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
364 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
368 struct map_struct *buf;
369 struct sum_struct *s;
371 struct file_struct *file = flist->files[i];
374 fprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
376 statret = link_stat(fname,&st);
378 if (S_ISDIR(file->mode)) {
380 if (statret == 0 && !S_ISDIR(st.st_mode)) {
381 if (do_unlink(fname) != 0) {
382 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
387 if (statret != 0 && mkdir(fname,file->mode) != 0 && errno != EEXIST) {
388 if (!(relative_paths && errno==ENOENT &&
389 create_directory_path(fname)==0 &&
390 mkdir(fname,file->mode)==0)) {
391 fprintf(FERROR,"mkdir %s : %s (2)\n",
392 fname,strerror(errno));
395 if (set_perms(fname,file,NULL,0) && verbose)
396 fprintf(FINFO,"%s/\n",fname);
400 if (preserve_links && S_ISLNK(file->mode)) {
402 char lnk[MAXPATHLEN];
405 l = readlink(fname,lnk,MAXPATHLEN-1);
408 if (strcmp(lnk,file->link) == 0) {
409 set_perms(fname,file,&st,1);
415 if (do_symlink(file->link,fname) != 0) {
416 fprintf(FERROR,"link %s -> %s : %s\n",
417 fname,file->link,strerror(errno));
419 set_perms(fname,file,NULL,0);
421 fprintf(FINFO,"%s -> %s\n",
429 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
431 st.st_mode != file->mode ||
432 st.st_rdev != file->rdev) {
435 fprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
436 fname,(int)file->mode,(int)file->rdev);
437 if (do_mknod(fname,file->mode,file->rdev) != 0) {
438 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
440 set_perms(fname,file,NULL,0);
442 fprintf(FINFO,"%s\n",fname);
445 set_perms(fname,file,&st,1);
451 if (preserve_hard_links && check_hard_link(file)) {
453 fprintf(FINFO,"%s is a hard link\n",f_name(file));
457 if (!S_ISREG(file->mode)) {
458 fprintf(FINFO,"skipping non-regular file %s\n",fname);
463 if (errno == ENOENT) {
465 if (!dry_run) send_sums(NULL,f_out);
468 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
473 if (!S_ISREG(st.st_mode)) {
474 if (delete_file(fname) != 0) {
478 /* now pretend the file didn't exist */
480 if (!dry_run) send_sums(NULL,f_out);
484 if (update_only && st.st_mtime > file->modtime) {
486 fprintf(FINFO,"%s is newer\n",fname);
490 if (skip_file(fname, file, &st)) {
491 set_perms(fname,file,&st,1);
502 send_sums(NULL,f_out);
507 fd = open(fname,O_RDONLY);
510 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
511 fprintf(FERROR,"skipping %s\n",fname);
515 if (st.st_size > 0) {
516 buf = map_file(fd,st.st_size);
522 fprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
524 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
527 fprintf(FINFO,"sending sums for %d\n",i);
534 if (buf) unmap_file(buf);
541 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
543 int i,n,remainder,len,count;
547 static char file_sum1[MD4_SUM_LENGTH];
548 static char file_sum2[MD4_SUM_LENGTH];
551 count = read_int(f_in);
553 remainder = read_int(f_in);
557 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
560 fprintf(FINFO,"data recv %d at %d\n",i,(int)offset);
564 if (fd != -1 && write_sparse(fd,data,i) != i) {
565 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
573 if (i == count-1 && remainder != 0)
577 fprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n",
578 i,len,(int)offset2,(int)offset);
580 map = map_ptr(buf,offset2,len);
585 if (fd != -1 && write_sparse(fd,map,len) != len) {
586 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
593 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
594 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
600 if (remote_version >= 14) {
601 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
603 fprintf(FINFO,"got file_sum\n");
604 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
611 static void delete_one(struct file_struct *f)
613 if (!S_ISDIR(f->mode)) {
614 if (do_unlink(f_name(f)) != 0) {
615 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
616 } else if (verbose) {
617 fprintf(FINFO,"deleting %s\n",f_name(f));
620 if (do_rmdir(f_name(f)) != 0) {
621 if (errno != ENOTEMPTY && errno != EEXIST)
622 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
623 } else if (verbose) {
624 fprintf(FINFO,"deleting directory %s\n",f_name(f));
630 /* yuck! This function wouldn't have been necessary if I had the sorting
631 algorithm right. Unfortunately fixing the sorting algorithm would introduce
632 a backward incompatibility as file list indexes are sent over the link.
634 The aim is to see if a directory has already had the deletion algorithm applied
635 to it (due to recursion), and if so to skip it. The bisection is to
636 prevent this being an n^2 algorithm */
637 static int delete_already_done(struct file_list *flist,int j)
643 if (j == 0) return 0;
645 name = strdup(f_name(flist->files[j]));
648 fprintf(FERROR,"out of memory in delete_already_done");
652 p = strrchr(name,'/');
659 while (low != high) {
660 int mid = (low+high)/2;
661 int ret = strcmp(f_name(flist->files[flist_up(flist, mid)]),name);
673 low = flist_up(flist, low);
675 if (strcmp(f_name(flist->files[low]),name) == 0) {
685 /* this deletes any files on the receiving side that are not present
686 on the sending side. For version 1.6.4 I have changed the behaviour
687 to match more closely what most people seem to expect of this option */
688 static void delete_files(struct file_list *flist)
690 struct file_list *local_file_list;
697 fprintf(FINFO,"IO error encountered - skipping file deletion\n");
701 for (j=0;j<flist->count;j++) {
702 char *name = f_name(flist->files[j]);
704 if (!S_ISDIR(flist->files[j]->mode)) continue;
706 if (delete_already_done(flist, j)) continue;
708 if (!(local_file_list = send_file_list(-1,1,&name)))
712 fprintf(FINFO,"deleting in %s\n", name);
714 for (i=local_file_list->count-1;i>=0;i--) {
715 if (!local_file_list->files[i]->basename) continue;
716 if (-1 == flist_find(flist,local_file_list->files[i])) {
717 delete_one(local_file_list->files[i]);
720 flist_free(local_file_list);
724 static char *cleanup_fname;
726 void exit_cleanup(int code)
729 do_unlink(cleanup_fname);
730 signal(SIGUSR1, SIG_IGN);
743 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
748 char fnametmp[MAXPATHLEN];
749 struct map_struct *buf;
751 struct file_struct *file;
756 fprintf(FINFO,"recv_files(%d) starting\n",flist->count);
759 if (recurse && delete_mode && !local_name && flist->count>0) {
767 if (phase==0 && remote_version >= 13) {
769 csum_length = SUM_LENGTH;
771 fprintf(FINFO,"recv_files phase=%d\n",phase);
779 file = flist->files[i];
780 fname = f_name(file);
786 if (!am_server && verbose)
787 printf("%s\n",fname);
792 fprintf(FINFO,"recv_files(%s)\n",fname);
795 fd1 = open(fname,O_RDONLY);
797 if (fd1 != -1 && fstat(fd1,&st) != 0) {
798 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
799 receive_data(f_in,NULL,-1,NULL);
804 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
805 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
806 receive_data(f_in,NULL,-1,NULL);
811 if (fd1 != -1 && st.st_size > 0) {
812 buf = map_file(fd1,st.st_size);
814 fprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size);
820 if (strlen(fname) > (MAXPATHLEN-8)) {
821 fprintf(FERROR,"filename too long\n");
822 if (buf) unmap_file(buf);
828 f = strrchr(fname,'/');
833 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
835 sprintf(fnametmp,"%s.XXXXXX",fname);
837 if (NULL == mktemp(fnametmp)) {
838 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
839 receive_data(f_in,buf,-1,NULL);
840 if (buf) unmap_file(buf);
844 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
845 if (fd2 == -1 && relative_paths && errno == ENOENT &&
846 create_directory_path(fnametmp) == 0) {
847 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
850 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
851 receive_data(f_in,buf,-1,NULL);
852 if (buf) unmap_file(buf);
857 cleanup_fname = fnametmp;
859 if (!am_server && verbose)
860 printf("%s\n",fname);
863 recv_ok = receive_data(f_in,buf,fd2,fname);
865 if (buf) unmap_file(buf);
872 fprintf(FINFO,"renaming %s to %s\n",fnametmp,fname);
875 char fnamebak[MAXPATHLEN];
876 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
877 fprintf(FERROR,"backup filename too long\n");
880 sprintf(fnamebak,"%s%s",fname,backup_suffix);
881 if (rename(fname,fnamebak) != 0 && errno != ENOENT) {
882 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
887 /* move tmp file over real file */
888 if (rename(fnametmp,fname) != 0) {
889 if (errno == EXDEV) {
890 /* rename failed on cross-filesystem link.
891 Copy the file instead. */
892 if (copy_file(fnametmp,fname, file->mode)) {
893 fprintf(FERROR,"copy %s -> %s : %s\n",
894 fnametmp,fname,strerror(errno));
896 set_perms(fname,file,NULL,0);
900 fprintf(FERROR,"rename %s -> %s : %s\n",
901 fnametmp,fname,strerror(errno));
905 set_perms(fname,file,NULL,0);
908 cleanup_fname = NULL;
912 if (csum_length == SUM_LENGTH) {
913 fprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
917 fprintf(FINFO,"redoing %s(%d)\n",fname,i);
923 if (preserve_hard_links)
924 do_hard_links(flist);
926 /* now we need to fix any directory permissions that were
927 modified during the transfer */
928 for (i = 0; i < flist->count; i++) {
929 struct file_struct *file = flist->files[i];
930 if (!file->basename || !S_ISDIR(file->mode)) continue;
931 recv_generator(f_name(file),flist,i,-1);
935 fprintf(FINFO,"recv_files finished\n");
942 void send_files(struct file_list *flist,int f_out,int f_in)
945 struct sum_struct *s;
946 struct map_struct *buf;
948 char fname[MAXPATHLEN];
950 struct file_struct *file;
955 fprintf(FINFO,"send_files starting\n");
957 setup_nonblocking(f_in,f_out);
962 if (phase==0 && remote_version >= 13) {
964 csum_length = SUM_LENGTH;
968 fprintf(FINFO,"send_files phase=%d\n",phase);
974 file = flist->files[i];
978 strncpy(fname,file->basedir,MAXPATHLEN-1);
979 fname[MAXPATHLEN-1] = 0;
980 if (strlen(fname) == MAXPATHLEN-1) {
982 fprintf(FERROR, "send_files failed on long-named directory %s\n",
987 offset = strlen(file->basedir)+1;
989 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
992 fprintf(FINFO,"send_files(%d,%s)\n",i,fname);
995 if (!am_server && verbose)
996 printf("%s\n",fname);
1001 s = receive_sums(f_in);
1004 fprintf(FERROR,"receive_sums failed\n");
1008 fd = open(fname,O_RDONLY);
1011 fprintf(FERROR,"send_files failed to open %s: %s\n",
1012 fname,strerror(errno));
1017 /* map the local file */
1018 if (fstat(fd,&st) != 0) {
1020 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1026 if (st.st_size > 0) {
1027 buf = map_file(fd,st.st_size);
1033 fprintf(FINFO,"send_files mapped %s of size %d\n",
1034 fname,(int)st.st_size);
1038 write_int(f_out,s->count);
1039 write_int(f_out,s->n);
1040 write_int(f_out,s->remainder);
1043 fprintf(FINFO,"calling match_sums %s\n",fname);
1045 if (!am_server && verbose)
1046 printf("%s\n",fname+offset);
1048 match_sums(f_out,s,buf,st.st_size);
1051 if (buf) unmap_file(buf);
1057 fprintf(FINFO,"sender finished %s\n",fname);
1061 fprintf(FINFO,"send files finished\n");
1065 write_int(f_out,-1);
1071 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1077 fprintf(FINFO,"generator starting pid=%d count=%d\n",
1078 (int)getpid(),flist->count);
1080 for (i = 0; i < flist->count; i++) {
1081 struct file_struct *file = flist->files[i];
1082 mode_t saved_mode = file->mode;
1083 if (!file->basename) continue;
1085 /* we need to ensure that any directories we create have writeable
1086 permissions initially so that we can create the files within
1087 them. This is then fixed after the files are transferred */
1088 if (!am_root && S_ISDIR(file->mode)) {
1089 file->mode |= S_IWUSR; /* user write */
1092 recv_generator(local_name?local_name:f_name(file),
1095 file->mode = saved_mode;
1099 csum_length = SUM_LENGTH;
1103 fprintf(FINFO,"generate_files phase=%d\n",phase);
1108 /* we expect to just sit around now, so don't exit on a timeout. If we
1109 really get a timeout then the other process should exit */
1112 if (remote_version >= 13) {
1113 /* in newer versions of the protocol the files can cycle through
1114 the system more than once to catch initial checksum errors */
1115 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1116 struct file_struct *file = flist->files[i];
1117 recv_generator(local_name?local_name:f_name(file),
1123 fprintf(FINFO,"generate_files phase=%d\n",phase);
1131 fprintf(FINFO,"generator wrote %ld\n",(long)write_total());