added resend logic

added checksum on all files

diff --git a/checksum.c b/checksum.c
index 8d805b6..ba81358 100644 (file)
--- a/checksum.c
+++ b/checksum.c
@@ -83,7 +83,8 @@ void get_checksum2(char *buf,int len,char *sum)
   for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
     MDupdate(&MD, buf1+i, CSUM_CHUNK*8);
   }
   for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
     MDupdate(&MD, buf1+i, CSUM_CHUNK*8);
   }

-  MDupdate(&MD, buf1+i, (len-i)*8);
+  if (len - i > 0)
+    MDupdate(&MD, buf1+i, (len-i)*8);

 
   sum_put(&MD,sum);
 }
 
   sum_put(&MD,sum);
 }


@@ -112,8 +113,10 @@ void file_checksum(char *fname,char *sum,off_t size)
     MDupdate(&MD, tmpchunk, CSUM_CHUNK*8);
   }
 
     MDupdate(&MD, tmpchunk, CSUM_CHUNK*8);
   }
 

-  bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
-  MDupdate(&MD, tmpchunk, (len-i)*8);
+  if (len - i > 0) {
+    bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
+    MDupdate(&MD, tmpchunk, (len-i)*8);
+  }

 
   sum_put(&MD,sum);
 
 
   sum_put(&MD,sum);
 


@@ -127,6 +130,53 @@ void checksum_init(void)
 }
 
 
 }
 
 

+
+static MDstruct sumMD;
+static int sumresidue;
+static char sumrbuf[CSUM_CHUNK];
+
+void sum_init(void)
+{
+  MDbegin(&sumMD);
+  sumresidue=0;
+}
+
+void sum_update(char *p,int len)
+{
+  int i;
+  if (sumresidue) {
+    i = MIN(CSUM_CHUNK-sumresidue,len);
+    bcopy(p,sumrbuf+sumresidue,i);
+    MDupdate(&sumMD, sumrbuf, (i+sumresidue)*8);
+    len -= i;
+    p += i;
+  }
+
+  for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
+    bcopy(p+i,sumrbuf,CSUM_CHUNK);
+    MDupdate(&sumMD, sumrbuf, CSUM_CHUNK*8);
+  }
+
+  if (len - i > 0) {
+    sumresidue = len-i;
+    bcopy(p+i,sumrbuf,sumresidue);
+  } else {
+    sumresidue = 0;    
+  }
+}
+
+void sum_end(char *sum)
+{
+  if (sumresidue)
+    MDupdate(&sumMD, sumrbuf, sumresidue*8);
+
+  SIVAL(sum,0,sumMD.buffer[0]);
+  SIVAL(sum,4,sumMD.buffer[1]);
+  SIVAL(sum,8,sumMD.buffer[2]);
+  SIVAL(sum,12,sumMD.buffer[3]);  
+}
+
+

 #ifdef CHECKSUM_MAIN
  int main(int argc,char *argv[])
 {
 #ifdef CHECKSUM_MAIN
  int main(int argc,char *argv[])
 {

diff --git a/io.c b/io.c
index 3c55dbe..b25225e 100644 (file)
--- a/io.c
+++ b/io.c
@@ -129,8 +129,8 @@ int read_int(int f)
   char b[4];
   if ((ret=readfd(f,b,4)) != 4) {
     if (verbose > 1) 
   char b[4];
   if ((ret=readfd(f,b,4)) != 4) {
     if (verbose > 1) 

-      fprintf(FERROR,"Error reading %d bytes : %s\n",
-             4,ret==-1?strerror(errno):"EOF");
+      fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+             getpid(),4,ret==-1?strerror(errno):"EOF");

     exit_cleanup(1);
   }
   total_read += 4;
     exit_cleanup(1);
   }
   total_read += 4;


@@ -142,8 +142,8 @@ void read_buf(int f,char *buf,int len)
   int ret;
   if ((ret=readfd(f,buf,len)) != len) {
     if (verbose > 1) 
   int ret;
   if ((ret=readfd(f,buf,len)) != len) {
     if (verbose > 1) 

-      fprintf(FERROR,"Error reading %d bytes : %s\n",
-             len,ret==-1?strerror(errno):"EOF");
+      fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+             getpid(),len,ret==-1?strerror(errno):"EOF");

     exit_cleanup(1);
   }
   total_read += len;
     exit_cleanup(1);
   }
   total_read += len;

diff --git a/match.c b/match.c
index d7af159..16df871 100644 (file)
--- a/match.c
+++ b/match.c
@@ -24,6 +24,8 @@ extern int csum_length;
 extern int verbose;
 extern int am_server;
 
 extern int verbose;
 extern int am_server;
 

+extern int remote_version;
+

 typedef unsigned short tag;
 
 #define TABLESIZE (1<<16)
 typedef unsigned short tag;
 
 #define TABLESIZE (1<<16)


@@ -93,23 +95,53 @@ static void matched(int f,struct sum_struct *s,struct map_struct *buf,
                    int offset,int i)
 {
   int n = offset - last_match;
                    int offset,int i)
 {
   int n = offset - last_match;

-  
+  int j;
+

   if (verbose > 2)
     if (i != -1)
       fprintf(FERROR,"match at %d last_match=%d j=%d len=%d n=%d\n",
              (int)offset,(int)last_match,i,(int)s->sums[i].len,n);
 
   if (verbose > 2)
     if (i != -1)
       fprintf(FERROR,"match at %d last_match=%d j=%d len=%d n=%d\n",
              (int)offset,(int)last_match,i,(int)s->sums[i].len,n);
 

-  send_token(f,i,buf,last_match,n);
-
+  send_token(f,i,buf,last_match,n,i==-1?0:s->sums[i].len);

   data_transfer += n;
 
   data_transfer += n;
 

+  if (n > 0)
+    write_flush(f);
+

   if (i != -1)
   if (i != -1)

-    last_match = offset + s->sums[i].len;
+    n += s->sums[i].len;

   
   

+  for (j=0;j<n;j+=CHUNK_SIZE) {
+    int n1 = MIN(CHUNK_SIZE,n-j);
+    sum_update(map_ptr(buf,last_match+j,n1),n1);
+  }
+
+

   if (i != -1)
     last_match = offset + s->sums[i].len;
   if (i != -1)
     last_match = offset + s->sums[i].len;

-  if (n > 0)
-    write_flush(f);
+
+}
+
+
+static inline char *window_ptr(struct map_struct *buf,int off,int len)
+{
+  static char *p=NULL;
+  static int p_len = 0;
+  static int p_off = 0;  
+
+  if (off == 0) {
+    p_off = 0;
+    p_len = CHUNK_SIZE;
+    p = map_ptr(buf,p_off,p_len);    
+  }
+
+  while (off+len > p_off+p_len) {
+    p_off += CHUNK_SIZE;
+    p_len = CHUNK_SIZE;
+    p = map_ptr(buf,p_off,p_len);  
+  }
+
+  return(p + (off-p_off));

 }
 
 
 }
 
 


@@ -127,7 +159,7 @@ static void hash_search(int f,struct sum_struct *s,
 
   k = MIN(len, s->n);
 
 
   k = MIN(len, s->n);
 

-  map = map_ptr(buf,0,k);
+  map = window_ptr(buf,0,k);

 
   sum = get_checksum1(map, k);
   s1 = sum & 0xFFFF;
 
   sum = get_checksum1(map, k);
   s1 = sum & 0xFFFF;


@@ -165,7 +197,7 @@ static void hash_search(int f,struct sum_struct *s,
 
          if (!done_csum2) {
            int l = MIN(s->n,len-offset);
 
          if (!done_csum2) {
            int l = MIN(s->n,len-offset);

-           map = map_ptr(buf,offset,l);
+           map = window_ptr(buf,offset,l);

            get_checksum2(map,l,sum2);
            done_csum2 = 1;
          }
            get_checksum2(map,l,sum2);
            done_csum2 = 1;
          }


@@ -173,7 +205,7 @@ static void hash_search(int f,struct sum_struct *s,
            matched(f,s,buf,len,offset,i);
            offset += s->sums[i].len - 1;
            k = MIN((len-offset), s->n);
            matched(f,s,buf,len,offset,i);
            offset += s->sums[i].len - 1;
            k = MIN((len-offset), s->n);

-           map = map_ptr(buf,offset,k);
+           map = window_ptr(buf,offset,k);

            sum = get_checksum1(map, k);
            s1 = sum & 0xFFFF;
            s2 = sum >> 16;
            sum = get_checksum1(map, k);
            s1 = sum & 0xFFFF;
            s2 = sum >> 16;


@@ -188,7 +220,7 @@ static void hash_search(int f,struct sum_struct *s,
     }
 
     /* Trim off the first byte from the checksum */
     }
 
     /* Trim off the first byte from the checksum */

-    map = map_ptr(buf,offset,k+1);
+    map = window_ptr(buf,offset,k+1);

     s1 -= map[0];
     s2 -= k * map[0];
 
     s1 -= map[0];
     s2 -= k * map[0];
 


@@ -203,17 +235,22 @@ static void hash_search(int f,struct sum_struct *s,
   } while (++offset < end);
 
   matched(f,s,buf,len,len,-1);
   } while (++offset < end);
 
   matched(f,s,buf,len,len,-1);

+  window_ptr(buf,len-1,1);

 }
 
 
 void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
 {
 }
 
 
 void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
 {

+  char file_sum[SUM_LENGTH];
+

   last_match = 0;
   false_alarms = 0;
   tag_hits = 0;
   matches=0;
   data_transfer=0;
 
   last_match = 0;
   false_alarms = 0;
   tag_hits = 0;
   matches=0;
   data_transfer=0;
 

+  sum_init();
+

   if (len > 0 && s->count>0) {
     build_hash_table(s);
 
   if (len > 0 && s->count>0) {
     build_hash_table(s);
 


@@ -228,6 +265,14 @@ void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
     matched(f,s,buf,len,len,-1);
   }
 
     matched(f,s,buf,len,len,-1);
   }
 

+  sum_end(file_sum);
+
+  if (remote_version >= 14) {
+    if (verbose > 2)
+      fprintf(FERROR,"sending file_sum\n");
+    write_buf(f,file_sum,SUM_LENGTH);
+  }
+

   if (targets) {
     free(targets);
     targets=NULL;
   if (targets) {
     free(targets);
     targets=NULL;

diff --git a/rsync.c b/rsync.c
index 3f3e8f9..647af67 100644 (file)
--- a/rsync.c
+++ b/rsync.c
@@ -262,7 +262,7 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
   struct file_struct *file = &flist->files[i];
 
   if (verbose > 2)
   struct file_struct *file = &flist->files[i];
 
   if (verbose > 2)

-    fprintf(FERROR,"recv_generator(%s)\n",fname);
+    fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);

 
   statret = lstat(fname,&st);
 
 
   statret = lstat(fname,&st);
 


@@ -383,10 +383,13 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
   }
 
   if (verbose > 3)
   }
 
   if (verbose > 3)

-    fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
+    fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);

 
   s = generate_sums(buf,st.st_size,block_size);
 
 
   s = generate_sums(buf,st.st_size,block_size);
 

+  if (verbose > 2)
+    fprintf(FERROR,"sending sums for %d\n",i);
+

   write_int(f_out,i);
   send_sums(s,f_out);
   write_flush(f_out);
   write_int(f_out,i);
   send_sums(s,f_out);
   write_flush(f_out);


@@ -399,22 +402,29 @@ void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
 
 
 
 
 
 

-static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
+static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)

 {
   int i,n,remainder,len,count;
   off_t offset = 0;
   off_t offset2;
   char *data;
 {
   int i,n,remainder,len,count;
   off_t offset = 0;
   off_t offset2;
   char *data;

+  static char file_sum1[SUM_LENGTH];
+  static char file_sum2[SUM_LENGTH];
+  char *map=NULL;

 
   count = read_int(f_in);
   n = read_int(f_in);
   remainder = read_int(f_in);
 
 
   count = read_int(f_in);
   n = read_int(f_in);
   remainder = read_int(f_in);
 

+  sum_init();
+

   for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
     if (i > 0) {
       if (verbose > 3)
        fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
 
   for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
     if (i > 0) {
       if (verbose > 3)
        fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
 

+      sum_update(data,i);
+

       if (write_sparse(fd,data,i) != i) {
        fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
        exit_cleanup(1);
       if (write_sparse(fd,data,i) != i) {
        fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
        exit_cleanup(1);


@@ -431,7 +441,11 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
        fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
                i,len,(int)offset2,(int)offset);
 
        fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
                i,len,(int)offset2,(int)offset);
 

-      if (write_sparse(fd,map_ptr(buf,offset2,len),len) != len) {
+      map = map_ptr(buf,offset2,len);
+
+      sum_update(map,len);
+
+      if (write_sparse(fd,map,len) != len) {

        fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
        exit_cleanup(1);
       }
        fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
        exit_cleanup(1);
       }


@@ -443,6 +457,17 @@ static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
     fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
     exit_cleanup(1);
   }
     fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
     exit_cleanup(1);
   }

+
+  sum_end(file_sum1);
+
+  if (remote_version >= 14) {
+    read_buf(f_in,file_sum2,SUM_LENGTH);
+    if (verbose > 2)
+      fprintf(FERROR,"got file_sum\n");
+    if (memcmp(file_sum1,file_sum2,SUM_LENGTH) != 0)
+      return 0;
+  }
+  return 1;

 }
 
 
 }
 
 


@@ -510,6 +535,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
   int i;
   struct file_struct *file;
   int phase=0;
   int i;
   struct file_struct *file;
   int phase=0;

+  int recv_ok;

 
   if (verbose > 2) {
     fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
 
   if (verbose > 2) {
     fprintf(FERROR,"recv_files(%d) starting\n",flist->count);


@@ -520,11 +546,14 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
   }
 
   while (1) 
   }
 
   while (1) 

-    {
+    {      

       i = read_int(f_in);
       if (i == -1) {
        if (phase==0 && remote_version >= 13) {
          phase++;
       i = read_int(f_in);
       if (i == -1) {
        if (phase==0 && remote_version >= 13) {
          phase++;

+         csum_length = SUM_LENGTH;
+         if (verbose > 2)
+           fprintf(FERROR,"recv_files phase=%d\n",phase);

          write_int(f_gen,-1);
          write_flush(f_gen);
          continue;
          write_int(f_gen,-1);
          write_flush(f_gen);
          continue;


@@ -564,13 +593,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
 
       if (fd1 != -1 && st.st_size > 0) {
        buf = map_file(fd1,st.st_size);
 
       if (fd1 != -1 && st.st_size > 0) {
        buf = map_file(fd1,st.st_size);

+       if (verbose > 2)
+         fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);

       } else {
        buf = NULL;
       }
 
       } else {
        buf = NULL;
       }
 

-      if (verbose > 2)
-       fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
-

       /* open tmp file */
       sprintf(fnametmp,"%s.XXXXXX",fname);
       if (NULL == mktemp(fnametmp)) {
       /* open tmp file */
       sprintf(fnametmp,"%s.XXXXXX",fname);
       if (NULL == mktemp(fnametmp)) {


@@ -589,7 +617,7 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
        printf("%s\n",fname);
 
       /* recv file data */
        printf("%s\n",fname);
 
       /* recv file data */

-      receive_data(f_in,buf,fd2,fname);
+      recv_ok = receive_data(f_in,buf,fd2,fname);

 
       if (fd1 != -1) {
        unmap_file(buf);
 
       if (fd1 != -1) {
        unmap_file(buf);


@@ -618,6 +646,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen)
       cleanup_fname = NULL;
 
       set_perms(fname,file,NULL,0);
       cleanup_fname = NULL;
 
       set_perms(fname,file,NULL,0);

+
+      if (!recv_ok) {
+       if (verbose > 1)
+         fprintf(FERROR,"redoing %s(%d)\n",fname,i);
+       write_int(f_gen,i);
+      }

     }
 
   if (verbose > 2)
     }
 
   if (verbose > 2)


@@ -651,8 +685,11 @@ off_t send_files(struct file_list *flist,int f_out,int f_in)
       if (i == -1) {
        if (phase==0 && remote_version >= 13) {
          phase++;
       if (i == -1) {
        if (phase==0 && remote_version >= 13) {
          phase++;

+         csum_length = SUM_LENGTH;

          write_int(f_out,-1);
          write_flush(f_out);
          write_int(f_out,-1);
          write_flush(f_out);

+         if (verbose > 2)
+           fprintf(FERROR,"send_files phase=%d\n",phase);

          continue;
        }
        break;
          continue;
        }
        break;


@@ -748,6 +785,7 @@ off_t send_files(struct file_list *flist,int f_out,int f_in)
 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 {
   int i;
 void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
 {
   int i;

+  int phase=0;

 
   if (verbose > 2)
     fprintf(FERROR,"generator starting pid=%d count=%d\n",
 
   if (verbose > 2)
     fprintf(FERROR,"generator starting pid=%d count=%d\n",


@@ -769,19 +807,27 @@ void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
                   flist,i,f);
   }
 
                   flist,i,f);
   }
 

+  phase++;
+  csum_length = SUM_LENGTH;
+  ignore_times=1;
+
+  if (verbose > 2)
+    fprintf(FERROR,"generate_files phase=%d\n",phase);
+

   write_int(f,-1);
   write_flush(f);
 
   if (remote_version >= 13) {
   write_int(f,-1);
   write_flush(f);
 
   if (remote_version >= 13) {

-    /* go to the full checksum if anything has failed so far */
-    csum_length = SUM_LENGTH;
-

     for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
       struct file_struct *file = &flist->files[i];
       recv_generator(local_name?local_name:file->name,
                     flist,i,f);    
     }
 
     for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
       struct file_struct *file = &flist->files[i];
       recv_generator(local_name?local_name:file->name,
                     flist,i,f);    
     }
 

+    phase++;
+    if (verbose > 2)
+      fprintf(FERROR,"generate_files phase=%d\n",phase);
+

     write_int(f,-1);
     write_flush(f);
   }
     write_int(f,-1);
     write_flush(f);
   }

diff --git a/rsync.h b/rsync.h
index 5fcd40c..9bd631a 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -34,7 +34,7 @@
 #define SAME_TIME (1<<7)
 
 /* update this if you make incompatible changes */
 #define SAME_TIME (1<<7)
 
 /* update this if you make incompatible changes */

-#define PROTOCOL_VERSION 13
+#define PROTOCOL_VERSION 14

 #define MIN_PROTOCOL_VERSION 10
 #define MAX_PROTOCOL_VERSION 20
 
 #define MIN_PROTOCOL_VERSION 10
 #define MAX_PROTOCOL_VERSION 20
 

diff --git a/token.c b/token.c
index 3d6899a..887e625 100644 (file)
--- a/token.c
+++ b/token.c
@@ -73,7 +73,8 @@ static void simple_send_token(int f,int token,
  * If token == -1 then we have reached EOF 
  * If n == 0 then don't send a buffer
  */
  * If token == -1 then we have reached EOF 
  * If n == 0 then don't send a buffer
  */

-void send_token(int f,int token,struct map_struct *buf,int offset,int n)
+void send_token(int f,int token,struct map_struct *buf,int offset,
+               int n,int toklen)

 {
   if (!do_compression) {
     simple_send_token(f,token,buf,offset,n);
 {
   if (!do_compression) {
     simple_send_token(f,token,buf,offset,n);