From 9e31c4824cabb1bf173f87cbd0abe93f25ea9598 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 2 Jul 1996 03:19:58 +0000 Subject: [PATCH] added resend logic added checksum on all files --- checksum.c | 56 ++++++++++++++++++++++++++++++++++++++++--- io.c | 8 +++---- match.c | 65 ++++++++++++++++++++++++++++++++++++++++++-------- rsync.c | 70 ++++++++++++++++++++++++++++++++++++++++++++---------- rsync.h | 2 +- token.c | 3 ++- 6 files changed, 173 insertions(+), 31 deletions(-) diff --git a/checksum.c b/checksum.c index 8d805b6a..ba813586 100644 --- a/checksum.c +++ b/checksum.c @@ -83,7 +83,8 @@ void get_checksum2(char *buf,int len,char *sum) for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { MDupdate(&MD, buf1+i, CSUM_CHUNK*8); } - MDupdate(&MD, buf1+i, (len-i)*8); + if (len - i > 0) + MDupdate(&MD, buf1+i, (len-i)*8); sum_put(&MD,sum); } @@ -112,8 +113,10 @@ void file_checksum(char *fname,char *sum,off_t size) MDupdate(&MD, tmpchunk, CSUM_CHUNK*8); } - bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i); - MDupdate(&MD, tmpchunk, (len-i)*8); + if (len - i > 0) { + bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i); + MDupdate(&MD, tmpchunk, (len-i)*8); + } sum_put(&MD,sum); @@ -127,6 +130,53 @@ void checksum_init(void) } + +static MDstruct sumMD; +static int sumresidue; +static char sumrbuf[CSUM_CHUNK]; + +void sum_init(void) +{ + MDbegin(&sumMD); + sumresidue=0; +} + +void sum_update(char *p,int len) +{ + int i; + if (sumresidue) { + i = MIN(CSUM_CHUNK-sumresidue,len); + bcopy(p,sumrbuf+sumresidue,i); + MDupdate(&sumMD, sumrbuf, (i+sumresidue)*8); + len -= i; + p += i; + } + + for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { + bcopy(p+i,sumrbuf,CSUM_CHUNK); + MDupdate(&sumMD, sumrbuf, CSUM_CHUNK*8); + } + + if (len - i > 0) { + sumresidue = len-i; + bcopy(p+i,sumrbuf,sumresidue); + } else { + sumresidue = 0; + } +} + +void sum_end(char *sum) +{ + if (sumresidue) + MDupdate(&sumMD, sumrbuf, sumresidue*8); + + SIVAL(sum,0,sumMD.buffer[0]); + SIVAL(sum,4,sumMD.buffer[1]); + SIVAL(sum,8,sumMD.buffer[2]); + SIVAL(sum,12,sumMD.buffer[3]); +} + + #ifdef CHECKSUM_MAIN int main(int argc,char *argv[]) { diff --git a/io.c b/io.c index 3c55dbee..b25225ed 100644 --- a/io.c +++ b/io.c @@ -129,8 +129,8 @@ int read_int(int f) char b[4]; if ((ret=readfd(f,b,4)) != 4) { if (verbose > 1) - fprintf(FERROR,"Error reading %d bytes : %s\n", - 4,ret==-1?strerror(errno):"EOF"); + fprintf(FERROR,"(%d) Error reading %d bytes : %s\n", + getpid(),4,ret==-1?strerror(errno):"EOF"); exit_cleanup(1); } total_read += 4; @@ -142,8 +142,8 @@ void read_buf(int f,char *buf,int len) int ret; if ((ret=readfd(f,buf,len)) != len) { if (verbose > 1) - fprintf(FERROR,"Error reading %d bytes : %s\n", - len,ret==-1?strerror(errno):"EOF"); + fprintf(FERROR,"(%d) Error reading %d bytes : %s\n", + getpid(),len,ret==-1?strerror(errno):"EOF"); exit_cleanup(1); } total_read += len; diff --git a/match.c b/match.c index d7af159b..16df8714 100644 --- a/match.c +++ b/match.c @@ -24,6 +24,8 @@ extern int csum_length; extern int verbose; extern int am_server; +extern int remote_version; + typedef unsigned short tag; #define TABLESIZE (1<<16) @@ -93,23 +95,53 @@ static void matched(int f,struct sum_struct *s,struct map_struct *buf, int offset,int i) { int n = offset - last_match; - + int j; + if (verbose > 2) if (i != -1) fprintf(FERROR,"match at %d last_match=%d j=%d len=%d n=%d\n", (int)offset,(int)last_match,i,(int)s->sums[i].len,n); - send_token(f,i,buf,last_match,n); - + send_token(f,i,buf,last_match,n,i==-1?0:s->sums[i].len); data_transfer += n; + if (n > 0) + write_flush(f); + if (i != -1) - last_match = offset + s->sums[i].len; + n += s->sums[i].len; + for (j=0;jsums[i].len; - if (n > 0) - write_flush(f); + +} + + +static inline char *window_ptr(struct map_struct *buf,int off,int len) +{ + static char *p=NULL; + static int p_len = 0; + static int p_off = 0; + + if (off == 0) { + p_off = 0; + p_len = CHUNK_SIZE; + p = map_ptr(buf,p_off,p_len); + } + + while (off+len > p_off+p_len) { + p_off += CHUNK_SIZE; + p_len = CHUNK_SIZE; + p = map_ptr(buf,p_off,p_len); + } + + return(p + (off-p_off)); } @@ -127,7 +159,7 @@ static void hash_search(int f,struct sum_struct *s, k = MIN(len, s->n); - map = map_ptr(buf,0,k); + map = window_ptr(buf,0,k); sum = get_checksum1(map, k); s1 = sum & 0xFFFF; @@ -165,7 +197,7 @@ static void hash_search(int f,struct sum_struct *s, if (!done_csum2) { int l = MIN(s->n,len-offset); - map = map_ptr(buf,offset,l); + map = window_ptr(buf,offset,l); get_checksum2(map,l,sum2); done_csum2 = 1; } @@ -173,7 +205,7 @@ static void hash_search(int f,struct sum_struct *s, matched(f,s,buf,len,offset,i); offset += s->sums[i].len - 1; k = MIN((len-offset), s->n); - map = map_ptr(buf,offset,k); + map = window_ptr(buf,offset,k); sum = get_checksum1(map, k); s1 = sum & 0xFFFF; s2 = sum >> 16; @@ -188,7 +220,7 @@ static void hash_search(int f,struct sum_struct *s, } /* Trim off the first byte from the checksum */ - map = map_ptr(buf,offset,k+1); + map = window_ptr(buf,offset,k+1); s1 -= map[0]; s2 -= k * map[0]; @@ -203,17 +235,22 @@ static void hash_search(int f,struct sum_struct *s, } while (++offset < end); matched(f,s,buf,len,len,-1); + window_ptr(buf,len-1,1); } void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len) { + char file_sum[SUM_LENGTH]; + last_match = 0; false_alarms = 0; tag_hits = 0; matches=0; data_transfer=0; + sum_init(); + if (len > 0 && s->count>0) { build_hash_table(s); @@ -228,6 +265,14 @@ void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len) matched(f,s,buf,len,len,-1); } + sum_end(file_sum); + + if (remote_version >= 14) { + if (verbose > 2) + fprintf(FERROR,"sending file_sum\n"); + write_buf(f,file_sum,SUM_LENGTH); + } + if (targets) { free(targets); targets=NULL; diff --git a/rsync.c b/rsync.c index 3f3e8f9d..647af677 100644 --- a/rsync.c +++ b/rsync.c @@ -262,7 +262,7 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out) struct file_struct *file = &flist->files[i]; if (verbose > 2) - fprintf(FERROR,"recv_generator(%s)\n",fname); + fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i); statret = lstat(fname,&st); @@ -383,10 +383,13 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out) } if (verbose > 3) - fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size); + fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size); s = generate_sums(buf,st.st_size,block_size); + if (verbose > 2) + fprintf(FERROR,"sending sums for %d\n",i); + write_int(f_out,i); send_sums(s,f_out); write_flush(f_out); @@ -399,22 +402,29 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out) -static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname) +static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname) { int i,n,remainder,len,count; off_t offset = 0; off_t offset2; char *data; + static char file_sum1[SUM_LENGTH]; + static char file_sum2[SUM_LENGTH]; + char *map=NULL; count = read_int(f_in); n = read_int(f_in); remainder = read_int(f_in); + sum_init(); + for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) { if (i > 0) { if (verbose > 3) fprintf(FERROR,"data recv %d at %d\n",i,(int)offset); + sum_update(data,i); + if (write_sparse(fd,data,i) != i) { fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno)); exit_cleanup(1); @@ -431,7 +441,11 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname) fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n", i,len,(int)offset2,(int)offset); - if (write_sparse(fd,map_ptr(buf,offset2,len),len) != len) { + map = map_ptr(buf,offset2,len); + + sum_update(map,len); + + if (write_sparse(fd,map,len) != len) { fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno)); exit_cleanup(1); } @@ -443,6 +457,17 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname) fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno)); exit_cleanup(1); } + + sum_end(file_sum1); + + if (remote_version >= 14) { + read_buf(f_in,file_sum2,SUM_LENGTH); + if (verbose > 2) + fprintf(FERROR,"got file_sum\n"); + if (memcmp(file_sum1,file_sum2,SUM_LENGTH) != 0) + return 0; + } + return 1; } @@ -510,6 +535,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) int i; struct file_struct *file; int phase=0; + int recv_ok; if (verbose > 2) { fprintf(FERROR,"recv_files(%d) starting\n",flist->count); @@ -520,11 +546,14 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) } while (1) - { + { i = read_int(f_in); if (i == -1) { if (phase==0 && remote_version >= 13) { phase++; + csum_length = SUM_LENGTH; + if (verbose > 2) + fprintf(FERROR,"recv_files phase=%d\n",phase); write_int(f_gen,-1); write_flush(f_gen); continue; @@ -564,13 +593,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) if (fd1 != -1 && st.st_size > 0) { buf = map_file(fd1,st.st_size); + if (verbose > 2) + fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size); } else { buf = NULL; } - if (verbose > 2) - fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size); - /* open tmp file */ sprintf(fnametmp,"%s.XXXXXX",fname); if (NULL == mktemp(fnametmp)) { @@ -589,7 +617,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) printf("%s\n",fname); /* recv file data */ - receive_data(f_in,buf,fd2,fname); + recv_ok = receive_data(f_in,buf,fd2,fname); if (fd1 != -1) { unmap_file(buf); @@ -618,6 +646,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) cleanup_fname = NULL; set_perms(fname,file,NULL,0); + + if (!recv_ok) { + if (verbose > 1) + fprintf(FERROR,"redoing %s(%d)\n",fname,i); + write_int(f_gen,i); + } } if (verbose > 2) @@ -651,8 +685,11 @@ off_t send_files(struct file_list *flist,int f_out,int f_in) if (i == -1) { if (phase==0 && remote_version >= 13) { phase++; + csum_length = SUM_LENGTH; write_int(f_out,-1); write_flush(f_out); + if (verbose > 2) + fprintf(FERROR,"send_files phase=%d\n",phase); continue; } break; @@ -748,6 +785,7 @@ off_t send_files(struct file_list *flist,int f_out,int f_in) void generate_files(int f,struct file_list *flist,char *local_name,int f_recv) { int i; + int phase=0; if (verbose > 2) fprintf(FERROR,"generator starting pid=%d count=%d\n", @@ -769,19 +807,27 @@ void generate_files(int f,struct file_list *flist,char *local_name,int f_recv) flist,i,f); } + phase++; + csum_length = SUM_LENGTH; + ignore_times=1; + + if (verbose > 2) + fprintf(FERROR,"generate_files phase=%d\n",phase); + write_int(f,-1); write_flush(f); if (remote_version >= 13) { - /* go to the full checksum if anything has failed so far */ - csum_length = SUM_LENGTH; - for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) { struct file_struct *file = &flist->files[i]; recv_generator(local_name?local_name:file->name, flist,i,f); } + phase++; + if (verbose > 2) + fprintf(FERROR,"generate_files phase=%d\n",phase); + write_int(f,-1); write_flush(f); } diff --git a/rsync.h b/rsync.h index 5fcd40c7..9bd631a1 100644 --- a/rsync.h +++ b/rsync.h @@ -34,7 +34,7 @@ #define SAME_TIME (1<<7) /* update this if you make incompatible changes */ -#define PROTOCOL_VERSION 13 +#define PROTOCOL_VERSION 14 #define MIN_PROTOCOL_VERSION 10 #define MAX_PROTOCOL_VERSION 20 diff --git a/token.c b/token.c index 3d6899ae..887e6254 100644 --- a/token.c +++ b/token.c @@ -73,7 +73,8 @@ static void simple_send_token(int f,int token, * If token == -1 then we have reached EOF * If n == 0 then don't send a buffer */ -void send_token(int f,int token,struct map_struct *buf,int offset,int n) +void send_token(int f,int token,struct map_struct *buf,int offset, + int n,int toklen) { if (!do_compression) { simple_send_token(f,token,buf,offset,n); -- 2.34.1