2 Copyright (C) Andrew Tridgell 1996
3 Copyright (C) Paul Mackerras 1996
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 extern int csum_length;
26 extern int always_checksum;
27 extern time_t starttime;
29 extern int remote_version;
31 extern char *backup_suffix;
34 extern int whole_file;
35 extern int block_size;
36 extern int update_only;
37 extern int make_backups;
38 extern int preserve_links;
39 extern int preserve_hard_links;
40 extern int preserve_perms;
41 extern int preserve_devices;
42 extern int preserve_uid;
43 extern int preserve_gid;
44 extern int preserve_times;
46 extern int ignore_times;
48 extern int delete_mode;
49 extern int cvs_exclude;
51 extern int relative_paths;
56 static void free_sums(struct sum_struct *s)
58 if (s->sums) free(s->sums);
65 send a sums struct down a fd
67 static void send_sums(struct sum_struct *s,int f_out)
71 /* tell the other guy how many we are going to be doing and how many
72 bytes there are in the last chunk */
73 write_int(f_out,s?s->count:0);
74 write_int(f_out,s?s->n:block_size);
75 write_int(f_out,s?s->remainder:0);
77 for (i=0;i<s->count;i++) {
78 write_int(f_out,s->sums[i].sum1);
79 write_buf(f_out,s->sums[i].sum2,csum_length);
86 generate a stream of signatures/checksums that describe a buffer
88 generate approximately one checksum every n bytes
90 static struct sum_struct *generate_sums(struct map_struct *buf,off_t len,int n)
96 int remainder = (len%block_len);
99 count = (len+(block_len-1))/block_len;
101 s = (struct sum_struct *)malloc(sizeof(*s));
102 if (!s) out_of_memory("generate_sums");
105 s->remainder = remainder;
115 fprintf(FERROR,"count=%d rem=%d n=%d flength=%d\n",
116 s->count,s->remainder,s->n,(int)s->flength);
118 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
119 if (!s->sums) out_of_memory("generate_sums");
121 for (i=0;i<count;i++) {
123 char *map = map_ptr(buf,offset,n1);
125 s->sums[i].sum1 = get_checksum1(map,n1);
126 get_checksum2(map,n1,s->sums[i].sum2);
128 s->sums[i].offset = offset;
133 fprintf(FERROR,"chunk[%d] offset=%d len=%d sum1=%08x\n",
134 i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
145 receive the checksums for a buffer
147 static struct sum_struct *receive_sums(int f)
149 struct sum_struct *s;
153 s = (struct sum_struct *)malloc(sizeof(*s));
154 if (!s) out_of_memory("receive_sums");
156 s->count = read_int(f);
158 s->remainder = read_int(f);
162 fprintf(FERROR,"count=%d n=%d rem=%d\n",
163 s->count,s->n,s->remainder);
168 s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
169 if (!s->sums) out_of_memory("receive_sums");
171 for (i=0;i<s->count;i++) {
172 s->sums[i].sum1 = read_int(f);
173 read_buf(f,s->sums[i].sum2,csum_length);
175 s->sums[i].offset = offset;
178 if (i == s->count-1 && s->remainder != 0) {
179 s->sums[i].len = s->remainder;
181 s->sums[i].len = s->n;
183 offset += s->sums[i].len;
186 fprintf(FERROR,"chunk[%d] len=%d offset=%d sum1=%08x\n",
187 i,s->sums[i].len,(int)s->sums[i].offset,s->sums[i].sum1);
196 static int set_perms(char *fname,struct file_struct *file,struct stat *st,
202 if (dry_run) return 0;
205 if (link_stat(fname,&st2) != 0) {
206 fprintf(FERROR,"stat %s : %s\n",fname,strerror(errno));
212 if (preserve_times && !S_ISLNK(st->st_mode) &&
213 st->st_mtime != file->modtime) {
215 if (set_modtime(fname,file->modtime) != 0) {
216 fprintf(FERROR,"failed to set times on %s : %s\n",
217 fname,strerror(errno));
223 if (preserve_perms && !S_ISLNK(st->st_mode) &&
224 st->st_mode != file->mode) {
226 if (chmod(fname,file->mode) != 0) {
227 fprintf(FERROR,"failed to set permissions on %s : %s\n",
228 fname,strerror(errno));
234 if ((am_root && preserve_uid && st->st_uid != file->uid) ||
235 (preserve_gid && st->st_gid != file->gid)) {
238 (am_root&&preserve_uid)?file->uid:-1,
239 preserve_gid?file->gid:-1) != 0) {
240 if (verbose>1 || preserve_uid)
241 fprintf(FERROR,"chown %s : %s\n",fname,strerror(errno));
246 if (verbose > 1 && report) {
248 fprintf(FINFO,"%s\n",fname);
250 fprintf(FINFO,"%s is uptodate\n",fname);
256 /* choose whether to skip a particular file */
257 static int skip_file(char *fname,
258 struct file_struct *file, struct stat *st)
260 if (st->st_size != file->length) {
264 /* if always checksum is set then we use the checksum instead
265 of the file time to determine whether to sync */
266 if (always_checksum && S_ISREG(st->st_mode)) {
267 char sum[MD4_SUM_LENGTH];
268 file_checksum(fname,sum,st->st_size);
269 return (memcmp(sum,file->sum,csum_length) == 0);
276 return (st->st_mtime == file->modtime);
280 /* use a larger block size for really big files */
281 int adapt_block_size(struct file_struct *file, int bsize)
283 int ret = file->length / (10000); /* rough heuristic */
284 ret = ret & ~15; /* multiple of 16 */
285 if (ret < bsize) ret = bsize;
286 if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
290 void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
294 struct map_struct *buf;
295 struct sum_struct *s;
297 struct file_struct *file = flist->files[i];
300 fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
302 statret = link_stat(fname,&st);
304 if (S_ISDIR(file->mode)) {
306 if (statret == 0 && !S_ISDIR(st.st_mode)) {
307 if (do_unlink(fname) != 0) {
308 fprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
313 if (statret != 0 && mkdir(fname,file->mode) != 0 && errno != EEXIST) {
314 if (!(relative_paths && errno==ENOENT &&
315 create_directory_path(fname)==0 &&
316 mkdir(fname,file->mode)==0)) {
317 fprintf(FERROR,"mkdir %s : %s (2)\n",
318 fname,strerror(errno));
321 if (set_perms(fname,file,NULL,0) && verbose)
322 fprintf(FINFO,"%s/\n",fname);
326 if (preserve_links && S_ISLNK(file->mode)) {
328 char lnk[MAXPATHLEN];
331 l = readlink(fname,lnk,MAXPATHLEN-1);
334 if (strcmp(lnk,file->link) == 0) {
335 set_perms(fname,file,&st,1);
341 if (do_symlink(file->link,fname) != 0) {
342 fprintf(FERROR,"link %s -> %s : %s\n",
343 fname,file->link,strerror(errno));
345 set_perms(fname,file,NULL,0);
347 fprintf(FINFO,"%s -> %s\n",
355 if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
357 st.st_mode != file->mode ||
358 st.st_rdev != file->rdev) {
361 fprintf(FERROR,"mknod(%s,0%o,0x%x)\n",
362 fname,(int)file->mode,(int)file->rdev);
363 if (do_mknod(fname,file->mode,file->rdev) != 0) {
364 fprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
366 set_perms(fname,file,NULL,0);
368 fprintf(FINFO,"%s\n",fname);
371 set_perms(fname,file,&st,1);
377 if (preserve_hard_links && check_hard_link(file)) {
379 fprintf(FINFO,"%s is a hard link\n",f_name(file));
383 if (!S_ISREG(file->mode)) {
384 fprintf(FERROR,"skipping non-regular file %s\n",fname);
389 if (errno == ENOENT) {
391 if (!dry_run) send_sums(NULL,f_out);
394 fprintf(FERROR,"recv_generator failed to open %s\n",fname);
399 if (!S_ISREG(st.st_mode)) {
400 /* its not a regular file on the receiving end, but it is on the
401 sending end. If its a directory then skip it (too dangerous to
402 do a recursive deletion??) otherwise try to unlink it */
403 if (S_ISDIR(st.st_mode)) {
404 fprintf(FERROR,"ERROR: %s is a directory\n",fname);
407 if (do_unlink(fname) != 0) {
408 fprintf(FERROR,"%s : not a regular file (generator)\n",fname);
412 /* now pretend the file didn't exist */
414 if (!dry_run) send_sums(NULL,f_out);
418 if (update_only && st.st_mtime > file->modtime) {
420 fprintf(FERROR,"%s is newer\n",fname);
424 if (skip_file(fname, file, &st)) {
425 set_perms(fname,file,&st,1);
436 send_sums(NULL,f_out);
441 fd = open(fname,O_RDONLY);
444 fprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
445 fprintf(FERROR,"skipping %s\n",fname);
449 if (st.st_size > 0) {
450 buf = map_file(fd,st.st_size);
456 fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
458 s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
461 fprintf(FERROR,"sending sums for %d\n",i);
468 if (buf) unmap_file(buf);
475 static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
477 int i,n,remainder,len,count;
481 static char file_sum1[MD4_SUM_LENGTH];
482 static char file_sum2[MD4_SUM_LENGTH];
485 count = read_int(f_in);
487 remainder = read_int(f_in);
491 for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
494 fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
498 if (fd != -1 && write_sparse(fd,data,i) != i) {
499 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
507 if (i == count-1 && remainder != 0)
511 fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
512 i,len,(int)offset2,(int)offset);
514 map = map_ptr(buf,offset2,len);
519 if (fd != -1 && write_sparse(fd,map,len) != len) {
520 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
527 if (fd != -1 && offset > 0 && sparse_end(fd) != 0) {
528 fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
534 if (remote_version >= 14) {
535 read_buf(f_in,file_sum2,MD4_SUM_LENGTH);
537 fprintf(FERROR,"got file_sum\n");
538 if (fd != -1 && memcmp(file_sum1,file_sum2,MD4_SUM_LENGTH) != 0)
545 static void delete_one(struct file_struct *f)
547 if (!S_ISDIR(f->mode)) {
548 if (do_unlink(f_name(f)) != 0) {
549 fprintf(FERROR,"unlink %s : %s\n",f_name(f),strerror(errno));
550 } else if (verbose) {
551 fprintf(FERROR,"deleting %s\n",f_name(f));
554 if (do_rmdir(f_name(f)) != 0) {
555 if (errno != ENOTEMPTY)
556 fprintf(FERROR,"rmdir %s : %s\n",f_name(f),strerror(errno));
557 } else if (verbose) {
558 fprintf(FERROR,"deleting directory %s\n",f_name(f));
564 /* yuck! This function wouldn't have been necessary if I had the sorting
565 algorithm right. Unfortunately fixing the sorting algorithm would introduce
566 a backward incompatibility as file list indexes are sent over the link.
568 The aim is to see if a directory has already had the deletion algorithm applied
569 to it (due to recursion), and if so to skip it. The bisection is to
570 prevent this being an n^2 algorithm */
571 static int delete_already_done(struct file_list *flist,int j)
577 if (j == 0) return 0;
579 name = strdup(f_name(flist->files[j]));
582 fprintf(FERROR,"out of memory in delete_already_done");
586 p = strrchr(name,'/');
593 while (low != high) {
594 int mid = (low+high)/2;
595 int ret = strcmp(f_name(flist->files[flist_up(flist, mid)]),name);
607 low = flist_up(flist, low);
609 if (strcmp(f_name(flist->files[low]),name) == 0) {
619 /* this deletes any files on the receiving side that are not present
620 on the sending side. For version 1.6.4 I have changed the behaviour
621 to match more closely what most people seem to expect of this option */
622 static void delete_files(struct file_list *flist)
624 struct file_list *local_file_list;
630 for (j=0;j<flist->count;j++) {
631 char *name = f_name(flist->files[j]);
633 if (!S_ISDIR(flist->files[j]->mode)) continue;
635 if (delete_already_done(flist, j)) continue;
637 if (!(local_file_list = send_file_list(-1,1,&name)))
641 fprintf(FINFO,"deleting in %s\n", name);
643 for (i=local_file_list->count-1;i>=0;i--) {
644 if (!local_file_list->files[i]->basename) continue;
645 if (-1 == flist_find(flist,local_file_list->files[i])) {
646 delete_one(local_file_list->files[i]);
649 flist_free(local_file_list);
653 static char *cleanup_fname;
655 void exit_cleanup(int code)
658 do_unlink(cleanup_fname);
659 signal(SIGUSR1, SIG_IGN);
672 int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
677 char fnametmp[MAXPATHLEN];
678 struct map_struct *buf;
680 struct file_struct *file;
685 fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
688 if (recurse && delete_mode && !local_name && flist->count>0) {
696 if (phase==0 && remote_version >= 13) {
698 csum_length = SUM_LENGTH;
700 fprintf(FERROR,"recv_files phase=%d\n",phase);
708 file = flist->files[i];
709 fname = f_name(file);
715 if (!am_server && verbose)
716 printf("%s\n",fname);
721 fprintf(FERROR,"recv_files(%s)\n",fname);
724 fd1 = open(fname,O_RDONLY);
726 if (fd1 != -1 && fstat(fd1,&st) != 0) {
727 fprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno));
728 receive_data(f_in,NULL,-1,NULL);
733 if (fd1 != -1 && !S_ISREG(st.st_mode)) {
734 fprintf(FERROR,"%s : not a regular file (recv_files)\n",fname);
735 receive_data(f_in,NULL,-1,NULL);
740 if (fd1 != -1 && st.st_size > 0) {
741 buf = map_file(fd1,st.st_size);
743 fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
749 if (strlen(fname) > (MAXPATHLEN-8)) {
750 fprintf(FERROR,"filename too long\n");
751 if (buf) unmap_file(buf);
757 f = strrchr(fname,'/');
762 sprintf(fnametmp,"%s/%s.XXXXXX",tmpdir,f);
764 sprintf(fnametmp,"%s.XXXXXX",fname);
766 if (NULL == mktemp(fnametmp)) {
767 fprintf(FERROR,"mktemp %s failed\n",fnametmp);
768 receive_data(f_in,buf,-1,NULL);
769 if (buf) unmap_file(buf);
773 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
774 if (fd2 == -1 && relative_paths && errno == ENOENT &&
775 create_directory_path(fnametmp) == 0) {
776 fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode);
779 fprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno));
780 receive_data(f_in,buf,-1,NULL);
781 if (buf) unmap_file(buf);
786 cleanup_fname = fnametmp;
788 if (!am_server && verbose)
789 printf("%s\n",fname);
792 recv_ok = receive_data(f_in,buf,fd2,fname);
794 if (buf) unmap_file(buf);
801 fprintf(FERROR,"renaming %s to %s\n",fnametmp,fname);
804 char fnamebak[MAXPATHLEN];
805 if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) {
806 fprintf(FERROR,"backup filename too long\n");
809 sprintf(fnamebak,"%s%s",fname,backup_suffix);
810 if (rename(fname,fnamebak) != 0 && errno != ENOENT) {
811 fprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno));
816 /* move tmp file over real file */
817 if (rename(fnametmp,fname) != 0) {
818 if (errno == EXDEV) {
819 /* rename failed on cross-filesystem link.
820 Copy the file instead. */
821 if (copy_file(fnametmp,fname, file->mode)) {
822 fprintf(FERROR,"copy %s -> %s : %s\n",
823 fnametmp,fname,strerror(errno));
825 set_perms(fname,file,NULL,0);
829 fprintf(FERROR,"rename %s -> %s : %s\n",
830 fnametmp,fname,strerror(errno));
834 set_perms(fname,file,NULL,0);
837 cleanup_fname = NULL;
841 if (csum_length == SUM_LENGTH) {
842 fprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
846 fprintf(FERROR,"redoing %s(%d)\n",fname,i);
852 if (preserve_hard_links)
853 do_hard_links(flist);
855 /* now we need to fix any directory permissions that were
856 modified during the transfer */
857 for (i = 0; i < flist->count; i++) {
858 struct file_struct *file = flist->files[i];
859 if (!file->basename || !S_ISDIR(file->mode)) continue;
860 recv_generator(f_name(file),flist,i,-1);
864 fprintf(FERROR,"recv_files finished\n");
871 off_t send_files(struct file_list *flist,int f_out,int f_in)
874 struct sum_struct *s;
875 struct map_struct *buf;
877 char fname[MAXPATHLEN];
880 struct file_struct *file;
884 fprintf(FERROR,"send_files starting\n");
886 setup_nonblocking(f_in,f_out);
891 if (phase==0 && remote_version >= 13) {
893 csum_length = SUM_LENGTH;
897 fprintf(FERROR,"send_files phase=%d\n",phase);
903 file = flist->files[i];
907 strncpy(fname,file->basedir,MAXPATHLEN-1);
908 fname[MAXPATHLEN-1] = 0;
909 if (strlen(fname) == MAXPATHLEN-1) {
910 fprintf(FERROR, "send_files failed on long-named directory %s\n",
916 strncat(fname,f_name(file),MAXPATHLEN-strlen(fname));
919 fprintf(FERROR,"send_files(%d,%s)\n",i,fname);
922 if (!am_server && verbose)
923 printf("%s\n",fname);
928 s = receive_sums(f_in);
930 fprintf(FERROR,"receive_sums failed\n");
934 fd = open(fname,O_RDONLY);
936 fprintf(FERROR,"send_files failed to open %s: %s\n",
937 fname,strerror(errno));
942 /* map the local file */
943 if (fstat(fd,&st) != 0) {
944 fprintf(FERROR,"fstat failed : %s\n",strerror(errno));
950 if (st.st_size > 0) {
951 buf = map_file(fd,st.st_size);
957 fprintf(FERROR,"send_files mapped %s of size %d\n",
958 fname,(int)st.st_size);
962 write_int(f_out,s->count);
963 write_int(f_out,s->n);
964 write_int(f_out,s->remainder);
967 fprintf(FERROR,"calling match_sums %s\n",fname);
969 if (!am_server && verbose)
970 printf("%s\n",fname);
972 match_sums(f_out,s,buf,st.st_size);
975 if (buf) unmap_file(buf);
981 fprintf(FERROR,"sender finished %s\n",fname);
987 fprintf(FERROR,"send files finished\n");
999 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
1005 fprintf(FERROR,"generator starting pid=%d count=%d\n",
1006 (int)getpid(),flist->count);
1008 for (i = 0; i < flist->count; i++) {
1009 struct file_struct *file = flist->files[i];
1010 mode_t saved_mode = file->mode;
1011 if (!file->basename) continue;
1013 /* we need to ensure that any directories we create have writeable
1014 permissions initially so that we can create the files within
1015 them. This is then fixed after the files are transferred */
1016 if (!am_root && S_ISDIR(file->mode)) {
1017 file->mode |= S_IWUSR; /* user write */
1020 recv_generator(local_name?local_name:f_name(file),
1023 file->mode = saved_mode;
1027 csum_length = SUM_LENGTH;
1031 fprintf(FERROR,"generate_files phase=%d\n",phase);
1036 if (remote_version >= 13) {
1037 /* in newer versions of the protocol the files can cycle through
1038 the system more than once to catch initial checksum errors */
1039 for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
1040 struct file_struct *file = flist->files[i];
1041 recv_generator(local_name?local_name:f_name(file),
1047 fprintf(FERROR,"generate_files phase=%d\n",phase);
1055 fprintf(FERROR,"generator wrote %ld\n",(long)write_total());