2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
56 static void free_sums(struct sum_struct *s)
58 if (s->sums) free(s->sums);
64 * delete a file or directory. If force_delet is set then delete
67 static int delete_file(char *fname)
72 extern int force_delete;
76 if (do_unlink(fname) == 0 || errno == ENOENT) return 0;
79 ret = lstat(fname, &st);
81 ret = stat(fname, &st);
84 fprintf(FERROR,"stat(%s) : %s\n", fname, strerror(errno));
88 if (!S_ISDIR(st.st_mode)) {
89 fprintf(FERROR,"unlink(%s) : %s\n", fname, strerror(errno));
93 if (do_rmdir(fname) == 0 || errno == ENOENT) return 0;
94 if (!force_delete || errno != ENOTEMPTY) {
95 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
99 /* now we do a recsursive delete on the directory ... */
102 fprintf(FERROR,"opendir(%s): %s\n",
103 fname,strerror(errno));
107 for (di=readdir(d); di; di=readdir(d)) {
108 if (strcmp(di->d_name,".")==0 ||
109 strcmp(di->d_name,"..")==0)
111 strncpy(buf, fname, (MAXPATHLEN-strlen(di->d_name))-2);
113 strcat(buf, di->d_name);
114 buf[MAXPATHLEN-1] = 0;
116 fprintf(FINFO,"deleting %s\n", buf);
117 if (delete_file(buf) != 0) {
125 if (do_rmdir(fname) != 0) {
126 fprintf(FERROR,"rmdir(%s) : %s\n", fname, strerror(errno));
134 send a sums struct down a fd
136 static void send_sums(struct sum_struct *s,int f_out)
140 /* tell the other guy how many we are going to be doing and how many
141 bytes there are in the last chunk */
142 write_int(f_out,s?s->count:0);
143 write_int(f_out,s?s->n:block_size);
144 write_int(f_out,s?s->remainder:0);
146 for (i=0;i<s->count;i++) {
147 write_int(f_out,s->sums[i].sum1);
148 write_buf(f_out,s->sums[i].sum2,csum_length);
155 generate a stream of signatures/checksums that describe a buffer
157 generate approximately one checksum every n bytes
159 static struct sum_struct *generate_sums(struct map_struct *buf,off_t len,int n)
162 struct sum_struct *s;
165 int remainder = (len%block_len);
168 count = (len+(block_len-1))/block_len;
170 s = (struct sum_struct *)malloc(sizeof(*s));
171 if (!s) out_of_memory("generate_sums");
174 s->remainder = remainder;
184 fprintf(FERROR,"count=%d rem=%d n=%d flength=%d\n",
185 s->count,s->remainder,s->n,(int)s->flength);
187 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
188 if (!s->sums) out_of_memory("generate_sums");
190 for (i=0;i<count;i++) {
192 char *map = map_ptr(buf,offset,n1);
194 s->sums[i].sum1 = get_checksum1(map,n1);
195 get_checksum2(map,n1,s->sums[i].sum2);
197 s->sums[i].offset = offset;
202 fprintf(FERROR,"chunk[%d] offset=%d len=%d sum1=%08x\n",
203 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
214 receive the checksums for a buffer
216 static struct sum_struct *receive_sums(int f)
218 struct sum_struct *s;
222 s = (struct sum_struct *)malloc(sizeof(*s));
223 if (!s) out_of_memory("receive_sums");
225 s->count = read_int(f);
227 s->remainder = read_int(f);
231 fprintf(FERROR,"count=%d n=%d rem=%d\n",
232 s->count,s->n,s->remainder);
237 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
238 if (!s->sums) out_of_memory("receive_sums");
240 for (i=0;i<s->count;i++) {
241 s->sums[i].sum1 = read_int(f);
242 read_buf(f,s->sums[i].sum2,csum_length);
244 s->sums[i].offset = offset;
247 if (i == s->count-1 && s->remainder != 0) {
248 s->sums[i].len = s->remainder;
250 s->sums[i].len = s->n;
252 offset += s->sums[i].len;
255 fprintf(FERROR,"chunk[%d] len=%d offset=%d sum1=%08x\n",
256 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
265 static int set_perms(char *fname,struct file_struct *file,struct stat *st,
271 if (dry_run) return 0;
274 if (link_stat(fname,&st2) != 0) {
275 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
281 if (preserve_times && !S_ISLNK(st->st_mode) &&
282 st->st_mtime != file->modtime) {
284 if (set_modtime(fname,file->modtime) != 0) {
285 fprintf(FERROR,"failed to set times on %s : %s\n",
286 fname,strerror(errno));
292 if (preserve_perms && !S_ISLNK(st->st_mode) &&
293 st->st_mode != file->mode) {
295 if (do_chmod(fname,file->mode) != 0) {
296 fprintf(FERROR,"failed to set permissions on %s : %s\n",
297 fname,strerror(errno));
303 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
304 (preserve_gid && st->st_gid != file->gid)) {
307 (am_root&&preserve_uid)?file->uid:-1,
308 preserve_gid?file->gid:-1) != 0) {
309 if (verbose>1 || preserve_uid)
310 fprintf(FERROR,"chown %s : %s\n",fname,strerror(errno));
315 if (verbose > 1 && report) {
317 fprintf(FINFO,"%s\n",fname);
319 fprintf(FINFO,"%s is uptodate\n",fname);
325 /* choose whether to skip a particular file */
326 static int skip_file(char *fname,
327 struct file_struct *file, struct stat *st)
329 if (st->st_size != file->length) {
333 /* if always checksum is set then we use the checksum instead
334 of the file time to determine whether to sync */
335 if (always_checksum && S_ISREG(st->st_mode)) {
336 char sum[MD4_SUM_LENGTH];
337 file_checksum(fname,sum,st->st_size);
338 return (memcmp(sum,file->sum,csum_length) == 0);
345 return (st->st_mtime == file->modtime);
349 /* use a larger block size for really big files */
350 int adapt_block_size(struct file_struct *file, int bsize)
352 int ret = file->length / (10000); /* rough heuristic */
353 ret = ret & ~15; /* multiple of 16 */
354 if (ret < bsize) ret = bsize;
355 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
359 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
363 struct map_struct *buf;
364 struct sum_struct *s;
366 struct file_struct *file = flist->files[i];
369 fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
371 statret = link_stat(fname,&st);
373 if (S_ISDIR(file->mode)) {
375 if (statret == 0 && !S_ISDIR(st.st_mode)) {
376 if (do_unlink(fname) != 0) {
377 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
382 if (statret != 0 && mkdir(fname,file->mode) != 0 && errno != EEXIST) {
383 if (!(relative_paths && errno==ENOENT &&
384 create_directory_path(fname)==0 &&
385 mkdir(fname,file->mode)==0)) {
386 fprintf(FERROR,"mkdir %s : %s (2)\n",
387 fname,strerror(errno));
390 if (set_perms(fname,file,NULL,0) && verbose)
391 fprintf(FINFO,"%s/\n",fname);
395 if (preserve_links && S_ISLNK(file->mode)) {
397 char lnk[MAXPATHLEN];
400 l = readlink(fname,lnk,MAXPATHLEN-1);
403 if (strcmp(lnk,file->link) == 0) {
404 set_perms(fname,file,&st,1);
410 if (do_symlink(file->link,fname) != 0) {
411 fprintf(FERROR,"link %s -> %s : %s\n",
412 fname,file->link,strerror(errno));
414 set_perms(fname,file,NULL,0);
416 fprintf(FINFO,"%s -> %s\n",
424 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
426 st.st_mode != file->mode ||
427 st.st_rdev != file->rdev) {
430 fprintf(FERROR,"mknod(%s,0%o,0x%x)\n",
431 fname,(int)file->mode,(int)file->rdev);
432 if (do_mknod(fname,file->mode,file->rdev) != 0) {
433 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
435 set_perms(fname,file,NULL,0);
437 fprintf(FINFO,"%s\n",fname);
440 set_perms(fname,file,&st,1);
446 if (preserve_hard_links && check_hard_link(file)) {
448 fprintf(FINFO,"%s is a hard link\n",f_name(file));
452 if (!S_ISREG(file->mode)) {
453 fprintf(FERROR,"skipping non-regular file %s\n",fname);
458 if (errno == ENOENT) {
460 if (!dry_run) send_sums(NULL,f_out);
463 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
468 if (!S_ISREG(st.st_mode)) {
469 if (delete_file(fname) != 0) {
473 /* now pretend the file didn't exist */
475 if (!dry_run) send_sums(NULL,f_out);
479 if (update_only && st.st_mtime > file->modtime) {
481 fprintf(FERROR,"%s is newer\n",fname);
485 if (skip_file(fname, file, &st)) {
486 set_perms(fname,file,&st,1);
497 send_sums(NULL,f_out);
502 fd = open(fname,O_RDONLY);
505 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
506 fprintf(FERROR,"skipping %s\n",fname);
510 if (st.st_size > 0) {
511 buf = map_file(fd,st.st_size);
517 fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
519 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
522 fprintf(FERROR,"sending sums for %d\n",i);
529 if (buf) unmap_file(buf);
536 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
538 int i,n,remainder,len,count;
542 static char file_sum1[MD4_SUM_LENGTH];
543 static char file_sum2[MD4_SUM_LENGTH];
546 count = read_int(f_in);
548 remainder = read_int(f_in);
552 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
555 fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
559 if (fd != -1 && write_sparse(fd,data,i) != i) {
560 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
568 if (i == count-1 && remainder != 0)
572 fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
573 i,len,(int)offset2,(int)offset);
575 map = map_ptr(buf,offset2,len);
580 if (fd != -1 && write_sparse(fd,map,len) != len) {
581 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
588 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
589 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
595 if (remote_version >= 14) {
596 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
598 fprintf(FERROR,"got file_sum\n");
599 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
606 static void delete_one(struct file_struct *f)
608 if (!S_ISDIR(f->mode)) {
609 if (do_unlink(f_name(f)) != 0) {
610 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
611 } else if (verbose) {
612 fprintf(FERROR,"deleting %s\n",f_name(f));
615 if (do_rmdir(f_name(f)) != 0) {
616 if (errno != ENOTEMPTY)
617 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
618 } else if (verbose) {
619 fprintf(FERROR,"deleting directory %s\n",f_name(f));
625 /* yuck! This function wouldn't have been necessary if I had the sorting
626 algorithm right. Unfortunately fixing the sorting algorithm would introduce
627 a backward incompatibility as file list indexes are sent over the link.
629 The aim is to see if a directory has already had the deletion algorithm applied
630 to it (due to recursion), and if so to skip it. The bisection is to
631 prevent this being an n^2 algorithm */
632 static int delete_already_done(struct file_list *flist,int j)
638 if (j == 0) return 0;
640 name = strdup(f_name(flist->files[j]));
643 fprintf(FERROR,"out of memory in delete_already_done");
647 p = strrchr(name,'/');
654 while (low != high) {
655 int mid = (low+high)/2;
656 int ret = strcmp(f_name(flist->files[flist_up(flist, mid)]),name);
668 low = flist_up(flist, low);
670 if (strcmp(f_name(flist->files[low]),name) == 0) {
680 /* this deletes any files on the receiving side that are not present
681 on the sending side. For version 1.6.4 I have changed the behaviour
682 to match more closely what most people seem to expect of this option */
683 static void delete_files(struct file_list *flist)
685 struct file_list *local_file_list;
691 for (j=0;j<flist->count;j++) {
692 char *name = f_name(flist->files[j]);
694 if (!S_ISDIR(flist->files[j]->mode)) continue;
696 if (delete_already_done(flist, j)) continue;
698 if (!(local_file_list = send_file_list(-1,1,&name)))
702 fprintf(FINFO,"deleting in %s\n", name);
704 for (i=local_file_list->count-1;i>=0;i--) {
705 if (!local_file_list->files[i]->basename) continue;
706 if (-1 == flist_find(flist,local_file_list->files[i])) {
707 delete_one(local_file_list->files[i]);
710 flist_free(local_file_list);
714 static char *cleanup_fname;
716 void exit_cleanup(int code)
719 do_unlink(cleanup_fname);
720 signal(SIGUSR1, SIG_IGN);
733 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
738 char fnametmp[MAXPATHLEN];
739 struct map_struct *buf;
741 struct file_struct *file;
746 fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
749 if (recurse && delete_mode && !local_name && flist->count>0) {
757 if (phase==0 && remote_version >= 13) {
759 csum_length = SUM_LENGTH;
761 fprintf(FERROR,"recv_files phase=%d\n",phase);
769 file = flist->files[i];
770 fname = f_name(file);
776 if (!am_server && verbose)
777 printf("%s\n",fname);
782 fprintf(FERROR,"recv_files(%s)\n",fname);
785 fd1 = open(fname,O_RDONLY);
787 if (fd1 != -1 && fstat(fd1,&st) != 0) {
788 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
789 receive_data(f_in,NULL,-1,NULL);
794 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
795 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
796 receive_data(f_in,NULL,-1,NULL);
801 if (fd1 != -1 && st.st_size > 0) {
802 buf = map_file(fd1,st.st_size);
804 fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
810 if (strlen(fname) > (MAXPATHLEN-8)) {
811 fprintf(FERROR,"filename too long\n");
812 if (buf) unmap_file(buf);
818 f = strrchr(fname,'/');
823 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
825 sprintf(fnametmp,"%s.XXXXXX",fname);
827 if (NULL == mktemp(fnametmp)) {
828 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
829 receive_data(f_in,buf,-1,NULL);
830 if (buf) unmap_file(buf);
834 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
835 if (fd2 == -1 && relative_paths && errno == ENOENT &&
836 create_directory_path(fnametmp) == 0) {
837 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
840 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
841 receive_data(f_in,buf,-1,NULL);
842 if (buf) unmap_file(buf);
847 cleanup_fname = fnametmp;
849 if (!am_server && verbose)
850 printf("%s\n",fname);
853 recv_ok = receive_data(f_in,buf,fd2,fname);
855 if (buf) unmap_file(buf);
862 fprintf(FERROR,"renaming %s to %s\n",fnametmp,fname);
865 char fnamebak[MAXPATHLEN];
866 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
867 fprintf(FERROR,"backup filename too long\n");
870 sprintf(fnamebak,"%s%s",fname,backup_suffix);
871 if (rename(fname,fnamebak) != 0 && errno != ENOENT) {
872 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
877 /* move tmp file over real file */
878 if (rename(fnametmp,fname) != 0) {
879 if (errno == EXDEV) {
880 /* rename failed on cross-filesystem link.
881 Copy the file instead. */
882 if (copy_file(fnametmp,fname, file->mode)) {
883 fprintf(FERROR,"copy %s -> %s : %s\n",
884 fnametmp,fname,strerror(errno));
886 set_perms(fname,file,NULL,0);
890 fprintf(FERROR,"rename %s -> %s : %s\n",
891 fnametmp,fname,strerror(errno));
895 set_perms(fname,file,NULL,0);
898 cleanup_fname = NULL;
902 if (csum_length == SUM_LENGTH) {
903 fprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
907 fprintf(FERROR,"redoing %s(%d)\n",fname,i);
913 if (preserve_hard_links)
914 do_hard_links(flist);
916 /* now we need to fix any directory permissions that were
917 modified during the transfer */
918 for (i = 0; i < flist->count; i++) {
919 struct file_struct *file = flist->files[i];
920 if (!file->basename || !S_ISDIR(file->mode)) continue;
921 recv_generator(f_name(file),flist,i,-1);
925 fprintf(FERROR,"recv_files finished\n");
932 void send_files(struct file_list *flist,int f_out,int f_in)
935 struct sum_struct *s;
936 struct map_struct *buf;
938 char fname[MAXPATHLEN];
940 struct file_struct *file;
945 fprintf(FERROR,"send_files starting\n");
947 setup_nonblocking(f_in,f_out);
952 if (phase==0 && remote_version >= 13) {
954 csum_length = SUM_LENGTH;
958 fprintf(FERROR,"send_files phase=%d\n",phase);
964 file = flist->files[i];
968 strncpy(fname,file->basedir,MAXPATHLEN-1);
969 fname[MAXPATHLEN-1] = 0;
970 if (strlen(fname) == MAXPATHLEN-1) {
971 fprintf(FERROR, "send_files failed on long-named directory %s\n",
976 offset = strlen(file->basedir)+1;
978 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
981 fprintf(FERROR,"send_files(%d,%s)\n",i,fname);
984 if (!am_server && verbose)
985 printf("%s\n",fname);
990 s = receive_sums(f_in);
992 fprintf(FERROR,"receive_sums failed\n");
996 fd = open(fname,O_RDONLY);
998 fprintf(FERROR,"send_files failed to open %s: %s\n",
999 fname,strerror(errno));
1004 /* map the local file */
1005 if (fstat(fd,&st) != 0) {
1006 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
1012 if (st.st_size > 0) {
1013 buf = map_file(fd,st.st_size);
1019 fprintf(FERROR,"send_files mapped %s of size %d\n",
1020 fname,(int)st.st_size);
1024 write_int(f_out,s->count);
1025 write_int(f_out,s->n);
1026 write_int(f_out,s->remainder);
1029 fprintf(FERROR,"calling match_sums %s\n",fname);
1031 if (!am_server && verbose)
1032 printf("%s\n",fname+offset);
1034 match_sums(f_out,s,buf,st.st_size);
1037 if (buf) unmap_file(buf);
1043 fprintf(FERROR,"sender finished %s\n",fname);
1047 fprintf(FERROR,"send files finished\n");
1051 write_int(f_out,-1);
1057 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1063 fprintf(FERROR,"generator starting pid=%d count=%d\n",
1064 (int)getpid(),flist->count);
1066 for (i = 0; i < flist->count; i++) {
1067 struct file_struct *file = flist->files[i];
1068 mode_t saved_mode = file->mode;
1069 if (!file->basename) continue;
1071 /* we need to ensure that any directories we create have writeable
1072 permissions initially so that we can create the files within
1073 them. This is then fixed after the files are transferred */
1074 if (!am_root && S_ISDIR(file->mode)) {
1075 file->mode |= S_IWUSR; /* user write */
1078 recv_generator(local_name?local_name:f_name(file),
1081 file->mode = saved_mode;
1085 csum_length = SUM_LENGTH;
1089 fprintf(FERROR,"generate_files phase=%d\n",phase);
1094 if (remote_version >= 13) {
1095 /* in newer versions of the protocol the files can cycle through
1096 the system more than once to catch initial checksum errors */
1097 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1098 struct file_struct *file = flist->files[i];
1099 recv_generator(local_name?local_name:f_name(file),
1105 fprintf(FERROR,"generate_files phase=%d\n",phase);
1113 fprintf(FERROR,"generator wrote %ld\n",(long)write_total());