static off_t last_match;
-static void matched(int f,struct sum_struct *s,char *buf,off_t len,int offset,int i)
+static void matched(int f,struct sum_struct *s,char *buf,off_t len,
+ int offset,int i)
{
int n = offset - last_match;
(int)offset,(int)last_match,i,(int)s->sums[i].len,n);
if (n > 0) {
+ int l = 0;
write_int(f,n);
- write_buf(f,buf+last_match,n);
+ while (l < n) {
+ int n1 = MIN(WRITE_BLOCK_SIZE,n-l);
+ write_buf(f,map_ptr(buf,last_match+l,n1),n1);
+ l += n1;
+ }
data_transfer += n;
}
write_int(f,-(i+1));
int offset,j,k;
int end;
char sum2[SUM_LENGTH];
- uint32 s1, s2, sum;
+ uint32 s1, s2, sum;
+ char *map;
if (verbose > 2)
fprintf(stderr,"hash search b=%d len=%d\n",s->n,(int)len);
k = MIN(len, s->n);
- sum = get_checksum1(buf, k);
+
+ map = map_ptr(buf,0,k);
+
+ sum = get_checksum1(map, k);
s1 = sum & 0xFFFF;
s2 = sum >> 16;
if (verbose > 3)
offset,j,i,sum);
if (!done_csum2) {
- get_checksum2(buf+offset,MIN(s->n,len-offset),sum2);
+ int l = MIN(s->n,len-offset);
+ map = map_ptr(buf,offset,l);
+ get_checksum2(map,l,sum2);
done_csum2 = 1;
}
if (memcmp(sum2,s->sums[i].sum2,SUM_LENGTH) == 0) {
matched(f,s,buf,len,offset,i);
offset += s->sums[i].len - 1;
k = MIN((len-offset), s->n);
- sum = get_checksum1(buf+offset, k);
+ map = map_ptr(buf,offset,k);
+ sum = get_checksum1(map, k);
s1 = sum & 0xFFFF;
s2 = sum >> 16;
++matches;
}
/* Trim off the first byte from the checksum */
- s1 -= buf[offset];
- s2 -= k * buf[offset];
+ map = map_ptr(buf,offset,k+1);
+ s1 -= map[0];
+ s2 -= k * map[0];
/* Add on the next byte (if there is one) to the checksum */
if (k < (len-offset)) {
- s1 += buf[offset+k];
+ s1 += map[k];
s2 += s1;
} else {
--k;
}
- if (verbose > 3)
- fprintf(stderr,"s2:s1 = %.4x%.4x sum=%.8x k=%d offset=%d took %x added %x\n",
- s2&0xffff, s1&0xffff, get_checksum1(buf+offset+1,k),
- k, (int)offset, buf[offset], buf[offset+k]);
} while (++offset < end);
matched(f,s,buf,len,len,-1);
for (i=0;i<count;i++) {
int n1 = MIN(len,n);
+ char *map = map_ptr(buf,offset,n1);
- s->sums[i].sum1 = get_checksum1(buf,n1);
- get_checksum2(buf,n1,s->sums[i].sum2);
+ s->sums[i].sum1 = get_checksum1(map,n1);
+ get_checksum2(map,n1,s->sums[i].sum2);
s->sums[i].offset = offset;
s->sums[i].len = n1;
i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
len -= n1;
- buf += n1;
offset += n1;
}
if (st.st_size > 0) {
buf = map_file(fd,st.st_size);
- if (!buf) {
- fprintf(stderr,"mmap : %s\n",strerror(errno));
- close(fd);
- return;
- }
} else {
buf = NULL;
}
fprintf(stderr,"chunk[%d] of size %d at %d offset=%d\n",
i,len,(int)offset2,(int)offset);
- if (write(fd,buf+offset2,len) != len) {
+ if (write(fd,map_ptr(buf,offset2,len),len) != len) {
fprintf(stderr,"write failed on %s : %s\n",fname,strerror(errno));
exit(1);
}
if (fd1 != -1 && st.st_size > 0) {
buf = map_file(fd1,st.st_size);
- if (!buf) {
- fprintf(stderr,"map_file failed\n");
- return -1;
- }
} else {
buf = NULL;
}
fprintf(stderr,"mktemp %s failed\n",fnametmp);
return -1;
}
- fd2 = open(fnametmp,O_WRONLY|O_CREAT,st.st_mode);
+ fd2 = open(fnametmp,O_WRONLY|O_CREAT,flist->files[i].mode);
if (fd2 == -1) {
fprintf(stderr,"open %s : %s\n",fnametmp,strerror(errno));
return -1;
if (st.st_size > 0) {
buf = map_file(fd,st.st_size);
- if (!buf) {
- fprintf(stderr,"map_file failed : %s\n",strerror(errno));
- return -1;
- }
} else {
buf = NULL;
}
/* block size to write files in */
#define WRITE_BLOCK_SIZE (32*1024)
+#define MAX_MAP_SIZE (4*1024*1024)
#define BLOCKING_TIMEOUT 10
return(len);
}
+static int map_fd = -1;
+static off_t map_size = 0;
+static char *p = NULL;
+static int p_size = 0;
+static int p_offset = 0;
+static int p_len = 0;
+
+
char *map_file(int fd,off_t len)
{
- char *ret = (char *)mmap(NULL,len,PROT_READ,MAP_SHARED,fd,0);
+ char *ret = NULL;
+#ifdef HAVE_MMAP
+ if (len < MAX_MAP_SIZE)
+ ret = (char *)mmap(NULL,len,PROT_READ,MAP_SHARED,fd,0);
+#endif
+ map_fd = fd;
+ map_size = len;
return ret;
}
+char *map_ptr(char *buf,off_t offset,int len)
+{
+ if (buf)
+ return buf+offset;
+
+ if (len == 0)
+ return NULL;
+
+ len = MIN(len,map_size-offset);
+
+ if (offset >= p_offset &&
+ offset+len <= p_offset+p_len) {
+ return (p + (offset - p_offset));
+ }
+
+ len = MAX(len,WRITE_BLOCK_SIZE);
+ len = MIN(len,map_size - offset);
+
+ if (len > p_size) {
+ if (p) free(p);
+ p = (char *)malloc(len);
+ if (!p) out_of_memory("map_ptr");
+ p_size = len;
+ }
+
+ if (lseek(map_fd,offset,SEEK_SET) != offset ||
+ read(map_fd,p,len) != len) {
+ fprintf(stderr,"EOF in map_ptr!\n");
+ exit(1);
+ }
+
+ p_offset = offset;
+ p_len = len;
+
+ return p;
+}
+
+
void unmap_file(char *buf,off_t len)
{
if (len > 0 && buf)
munmap(buf,len);
+ map_fd = -1;
+ map_size = 0;
+ p_len = 0;
}