for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
MDupdate(&MD, buf1+i, CSUM_CHUNK*8);
}
- MDupdate(&MD, buf1+i, (len-i)*8);
+ if (len - i > 0)
+ MDupdate(&MD, buf1+i, (len-i)*8);
sum_put(&MD,sum);
}
MDupdate(&MD, tmpchunk, CSUM_CHUNK*8);
}
- bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
- MDupdate(&MD, tmpchunk, (len-i)*8);
+ if (len - i > 0) {
+ bcopy(map_ptr(buf,i,len-i),tmpchunk,len-i);
+ MDupdate(&MD, tmpchunk, (len-i)*8);
+ }
sum_put(&MD,sum);
}
+
+static MDstruct sumMD;
+static int sumresidue;
+static char sumrbuf[CSUM_CHUNK];
+
+void sum_init(void)
+{
+ MDbegin(&sumMD);
+ sumresidue=0;
+}
+
+void sum_update(char *p,int len)
+{
+ int i;
+ if (sumresidue) {
+ i = MIN(CSUM_CHUNK-sumresidue,len);
+ bcopy(p,sumrbuf+sumresidue,i);
+ MDupdate(&sumMD, sumrbuf, (i+sumresidue)*8);
+ len -= i;
+ p += i;
+ }
+
+ for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
+ bcopy(p+i,sumrbuf,CSUM_CHUNK);
+ MDupdate(&sumMD, sumrbuf, CSUM_CHUNK*8);
+ }
+
+ if (len - i > 0) {
+ sumresidue = len-i;
+ bcopy(p+i,sumrbuf,sumresidue);
+ } else {
+ sumresidue = 0;
+ }
+}
+
+void sum_end(char *sum)
+{
+ if (sumresidue)
+ MDupdate(&sumMD, sumrbuf, sumresidue*8);
+
+ SIVAL(sum,0,sumMD.buffer[0]);
+ SIVAL(sum,4,sumMD.buffer[1]);
+ SIVAL(sum,8,sumMD.buffer[2]);
+ SIVAL(sum,12,sumMD.buffer[3]);
+}
+
+
#ifdef CHECKSUM_MAIN
int main(int argc,char *argv[])
{
char b[4];
if ((ret=readfd(f,b,4)) != 4) {
if (verbose > 1)
- fprintf(FERROR,"Error reading %d bytes : %s\n",
- 4,ret==-1?strerror(errno):"EOF");
+ fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+ getpid(),4,ret==-1?strerror(errno):"EOF");
exit_cleanup(1);
}
total_read += 4;
int ret;
if ((ret=readfd(f,buf,len)) != len) {
if (verbose > 1)
- fprintf(FERROR,"Error reading %d bytes : %s\n",
- len,ret==-1?strerror(errno):"EOF");
+ fprintf(FERROR,"(%d) Error reading %d bytes : %s\n",
+ getpid(),len,ret==-1?strerror(errno):"EOF");
exit_cleanup(1);
}
total_read += len;
extern int verbose;
extern int am_server;
+extern int remote_version;
+
typedef unsigned short tag;
#define TABLESIZE (1<<16)
int offset,int i)
{
int n = offset - last_match;
-
+ int j;
+
if (verbose > 2)
if (i != -1)
fprintf(FERROR,"match at %d last_match=%d j=%d len=%d n=%d\n",
(int)offset,(int)last_match,i,(int)s->sums[i].len,n);
- send_token(f,i,buf,last_match,n);
-
+ send_token(f,i,buf,last_match,n,i==-1?0:s->sums[i].len);
data_transfer += n;
+ if (n > 0)
+ write_flush(f);
+
if (i != -1)
- last_match = offset + s->sums[i].len;
+ n += s->sums[i].len;
+ for (j=0;j<n;j+=CHUNK_SIZE) {
+ int n1 = MIN(CHUNK_SIZE,n-j);
+ sum_update(map_ptr(buf,last_match+j,n1),n1);
+ }
+
+
if (i != -1)
last_match = offset + s->sums[i].len;
- if (n > 0)
- write_flush(f);
+
+}
+
+
+static inline char *window_ptr(struct map_struct *buf,int off,int len)
+{
+ static char *p=NULL;
+ static int p_len = 0;
+ static int p_off = 0;
+
+ if (off == 0) {
+ p_off = 0;
+ p_len = CHUNK_SIZE;
+ p = map_ptr(buf,p_off,p_len);
+ }
+
+ while (off+len > p_off+p_len) {
+ p_off += CHUNK_SIZE;
+ p_len = CHUNK_SIZE;
+ p = map_ptr(buf,p_off,p_len);
+ }
+
+ return(p + (off-p_off));
}
k = MIN(len, s->n);
- map = map_ptr(buf,0,k);
+ map = window_ptr(buf,0,k);
sum = get_checksum1(map, k);
s1 = sum & 0xFFFF;
if (!done_csum2) {
int l = MIN(s->n,len-offset);
- map = map_ptr(buf,offset,l);
+ map = window_ptr(buf,offset,l);
get_checksum2(map,l,sum2);
done_csum2 = 1;
}
matched(f,s,buf,len,offset,i);
offset += s->sums[i].len - 1;
k = MIN((len-offset), s->n);
- map = map_ptr(buf,offset,k);
+ map = window_ptr(buf,offset,k);
sum = get_checksum1(map, k);
s1 = sum & 0xFFFF;
s2 = sum >> 16;
}
/* Trim off the first byte from the checksum */
- map = map_ptr(buf,offset,k+1);
+ map = window_ptr(buf,offset,k+1);
s1 -= map[0];
s2 -= k * map[0];
} while (++offset < end);
matched(f,s,buf,len,len,-1);
+ window_ptr(buf,len-1,1);
}
void match_sums(int f,struct sum_struct *s,struct map_struct *buf,off_t len)
{
+ char file_sum[SUM_LENGTH];
+
last_match = 0;
false_alarms = 0;
tag_hits = 0;
matches=0;
data_transfer=0;
+ sum_init();
+
if (len > 0 && s->count>0) {
build_hash_table(s);
matched(f,s,buf,len,len,-1);
}
+ sum_end(file_sum);
+
+ if (remote_version >= 14) {
+ if (verbose > 2)
+ fprintf(FERROR,"sending file_sum\n");
+ write_buf(f,file_sum,SUM_LENGTH);
+ }
+
if (targets) {
free(targets);
targets=NULL;
struct file_struct *file = &flist->files[i];
if (verbose > 2)
- fprintf(FERROR,"recv_generator(%s)\n",fname);
+ fprintf(FERROR,"recv_generator(%s,%d)\n",fname,i);
statret = lstat(fname,&st);
}
if (verbose > 3)
- fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
+ fprintf(FERROR,"gen mapped %s of size %d\n",fname,(int)st.st_size);
s = generate_sums(buf,st.st_size,block_size);
+ if (verbose > 2)
+ fprintf(FERROR,"sending sums for %d\n",i);
+
write_int(f_out,i);
send_sums(s,f_out);
write_flush(f_out);
-static void receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
+static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname)
{
int i,n,remainder,len,count;
off_t offset = 0;
off_t offset2;
char *data;
+ static char file_sum1[SUM_LENGTH];
+ static char file_sum2[SUM_LENGTH];
+ char *map=NULL;
count = read_int(f_in);
n = read_int(f_in);
remainder = read_int(f_in);
+ sum_init();
+
for (i=recv_token(f_in,&data); i != 0; i=recv_token(f_in,&data)) {
if (i > 0) {
if (verbose > 3)
fprintf(FERROR,"data recv %d at %d\n",i,(int)offset);
+ sum_update(data,i);
+
if (write_sparse(fd,data,i) != i) {
fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
exit_cleanup(1);
fprintf(FERROR,"chunk[%d] of size %d at %d offset=%d\n",
i,len,(int)offset2,(int)offset);
- if (write_sparse(fd,map_ptr(buf,offset2,len),len) != len) {
+ map = map_ptr(buf,offset2,len);
+
+ sum_update(map,len);
+
+ if (write_sparse(fd,map,len) != len) {
fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
exit_cleanup(1);
}
fprintf(FERROR,"write failed on %s : %s\n",fname,strerror(errno));
exit_cleanup(1);
}
+
+ sum_end(file_sum1);
+
+ if (remote_version >= 14) {
+ read_buf(f_in,file_sum2,SUM_LENGTH);
+ if (verbose > 2)
+ fprintf(FERROR,"got file_sum\n");
+ if (memcmp(file_sum1,file_sum2,SUM_LENGTH) != 0)
+ return 0;
+ }
+ return 1;
}
int i;
struct file_struct *file;
int phase=0;
+ int recv_ok;
if (verbose > 2) {
fprintf(FERROR,"recv_files(%d) starting\n",flist->count);
}
while (1)
- {
+ {
i = read_int(f_in);
if (i == -1) {
if (phase==0 && remote_version >= 13) {
phase++;
+ csum_length = SUM_LENGTH;
+ if (verbose > 2)
+ fprintf(FERROR,"recv_files phase=%d\n",phase);
write_int(f_gen,-1);
write_flush(f_gen);
continue;
if (fd1 != -1 && st.st_size > 0) {
buf = map_file(fd1,st.st_size);
+ if (verbose > 2)
+ fprintf(FERROR,"recv mapped %s of size %d\n",fname,(int)st.st_size);
} else {
buf = NULL;
}
- if (verbose > 2)
- fprintf(FERROR,"mapped %s of size %d\n",fname,(int)st.st_size);
-
/* open tmp file */
sprintf(fnametmp,"%s.XXXXXX",fname);
if (NULL == mktemp(fnametmp)) {
printf("%s\n",fname);
/* recv file data */
- receive_data(f_in,buf,fd2,fname);
+ recv_ok = receive_data(f_in,buf,fd2,fname);
if (fd1 != -1) {
unmap_file(buf);
cleanup_fname = NULL;
set_perms(fname,file,NULL,0);
+
+ if (!recv_ok) {
+ if (verbose > 1)
+ fprintf(FERROR,"redoing %s(%d)\n",fname,i);
+ write_int(f_gen,i);
+ }
}
if (verbose > 2)
if (i == -1) {
if (phase==0 && remote_version >= 13) {
phase++;
+ csum_length = SUM_LENGTH;
write_int(f_out,-1);
write_flush(f_out);
+ if (verbose > 2)
+ fprintf(FERROR,"send_files phase=%d\n",phase);
continue;
}
break;
void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
{
int i;
+ int phase=0;
if (verbose > 2)
fprintf(FERROR,"generator starting pid=%d count=%d\n",
flist,i,f);
}
+ phase++;
+ csum_length = SUM_LENGTH;
+ ignore_times=1;
+
+ if (verbose > 2)
+ fprintf(FERROR,"generate_files phase=%d\n",phase);
+
write_int(f,-1);
write_flush(f);
if (remote_version >= 13) {
- /* go to the full checksum if anything has failed so far */
- csum_length = SUM_LENGTH;
-
for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
struct file_struct *file = &flist->files[i];
recv_generator(local_name?local_name:file->name,
flist,i,f);
}
+ phase++;
+ if (verbose > 2)
+ fprintf(FERROR,"generate_files phase=%d\n",phase);
+
write_int(f,-1);
write_flush(f);
}
#define SAME_TIME (1<<7)
/* update this if you make incompatible changes */
-#define PROTOCOL_VERSION 13
+#define PROTOCOL_VERSION 14
#define MIN_PROTOCOL_VERSION 10
#define MAX_PROTOCOL_VERSION 20
* If token == -1 then we have reached EOF
* If n == 0 then don't send a buffer
*/
-void send_token(int f,int token,struct map_struct *buf,int offset,int n)
+void send_token(int f,int token,struct map_struct *buf,int offset,
+ int n,int toklen)
{
if (!do_compression) {
simple_send_token(f,token,buf,offset,n);