this fixes two problems:

1) handle 64 bit file offsets in the token code. I wonder how large
bit files worked up till now?

2) send a null token when we have passed over a large lump of data
without finding a token match. This reduces the number of IOs
considerably as it removes the need for seeks/reads on the checksum
calculation and literal send code. This is not enabled yet for the
compressed case as the deflate token code can't handle it yet.

diff --git a/match.c b/match.c
index bc759e2..0d8bc31 100644 (file)
--- a/match.c
+++ b/match.c
@@ -100,7 +100,7 @@ static void matched(int f,struct sum_struct *s,struct map_struct *buf,
                rprintf(FINFO,"match at %d last_match=%d j=%d len=%d n=%d\n",
                        (int)offset,(int)last_match,i,(int)s->sums[i].len,(int)n);
 
                rprintf(FINFO,"match at %d last_match=%d j=%d len=%d n=%d\n",
                        (int)offset,(int)last_match,i,(int)s->sums[i].len,(int)n);
 

-       send_token(f,i,buf,last_match,n,i==-1?0:s->sums[i].len);
+       send_token(f,i,buf,last_match,n,i<0?0:s->sums[i].len);

        data_transfer += n;
 
        if (n > 0)
        data_transfer += n;
 
        if (n > 0)


@@ -131,6 +131,7 @@ static void hash_search(int f,struct sum_struct *s,
        char sum2[SUM_LENGTH];
        uint32 s1, s2, sum; 
        schar *map;
        char sum2[SUM_LENGTH];
        uint32 s1, s2, sum; 
        schar *map;

+       extern int do_compression;

 
        if (verbose > 2)
                rprintf(FINFO,"hash search b=%d len=%d\n",s->n,(int)len);
 
        if (verbose > 2)
                rprintf(FINFO,"hash search b=%d len=%d\n",s->n,(int)len);


@@ -212,7 +213,23 @@ static void hash_search(int f,struct sum_struct *s,
                } else {
                        --k;
                }
                } else {
                        --k;
                }

-               
+
+               if (!do_compression) {
+                       /* By matching early we avoid re-reading the
+                          data 3 times in the case where a token
+                          match comes a long way after last
+                          match. The 3 reads are caused by the
+                          running match, the checksum update and the
+                          literal send.
+
+                          we don't enable this for the compressed
+                          case yet as the deflated token code can't
+                          handle it. Paul is working on it */
+                       if (offset-last_match >= CHUNK_SIZE+s->n && 
+                           (end-offset > CHUNK_SIZE)) {
+                               matched(f,s,buf,offset - s->n, -2);
+                       }
+               }

        } while (++offset < end);
        
        matched(f,s,buf,len,-1);
        } while (++offset < end);
        
        matched(f,s,buf,len,-1);

diff --git a/token.c b/token.c
index 543031e..318bdbc 100644 (file)
--- a/token.c
+++ b/token.c
@@ -51,7 +51,7 @@ static int simple_recv_token(int f,char **data)
 
 /* non-compressing send token */
 static void simple_send_token(int f,int token,
 
 /* non-compressing send token */
 static void simple_send_token(int f,int token,

-                             struct map_struct *buf,int offset,int n)
+                             struct map_struct *buf,OFF_T offset,int n)

 {
        if (n > 0) {
                int l = 0;
 {
        if (n > 0) {
                int l = 0;


@@ -62,7 +62,10 @@ static void simple_send_token(int f,int token,
                        l += n1;
                }
        }
                        l += n1;
                }
        }

-       write_int(f,-(token+1));
+       /* a -2 token means to send data only and no token */
+       if (token != -2) {
+               write_int(f,-(token+1));
+       }

 }
 
 
 }
 
 


@@ -103,7 +106,7 @@ static char *obuf;
 /* Send a deflated token */
 static void
 send_deflated_token(int f, int token,
 /* Send a deflated token */
 static void
 send_deflated_token(int f, int token,

-                   struct map_struct *buf, int offset, int nb, int toklen)
+                   struct map_struct *buf, OFF_T offset, int nb, int toklen)

 {
     int n, r;
     static int init_done;
 {
     int n, r;
     static int init_done;


@@ -350,7 +353,7 @@ see_deflate_token(char *buf, int len)
  * If token == -1 then we have reached EOF 
  * If n == 0 then don't send a buffer
  */
  * If token == -1 then we have reached EOF 
  * If n == 0 then don't send a buffer
  */

-void send_token(int f,int token,struct map_struct *buf,int offset,
+void send_token(int f,int token,struct map_struct *buf,OFF_T offset,

                int n,int toklen)
 {
   if (!do_compression) {
                int n,int toklen)
 {
   if (!do_compression) {