From: Andrew Tridgell Date: Wed, 26 Jun 1996 01:35:13 +0000 (+0000) Subject: went to full non-blocking writes for the send_files() process X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/commitdiff_plain/720b47f2611d277684b2ec35d11dc3feedb207fe went to full non-blocking writes for the send_files() process --- diff --git a/.cvsignore b/.cvsignore index 071bfbad..d24a5970 100644 --- a/.cvsignore +++ b/.cvsignore @@ -39,6 +39,9 @@ rsync-0.8 rsync-0.8.tar.gz rsync-0.9 rsync-0.9.tar.gz +rsync-1.0 +rsync-1.1 +rsync-1.1.tar.gz rsync-ERSION rsync.aux rsync.dvi @@ -48,4 +51,3 @@ tech_report.dvi tech_report.log tech_report.ps test -rsync-1.0 diff --git a/Makefile.in b/Makefile.in index 9402dc8c..a4366420 100644 --- a/Makefile.in +++ b/Makefile.in @@ -20,7 +20,8 @@ SHELL=/bin/sh .SUFFIXES: .c .o LIBOBJ=lib/getopt.o lib/fnmatch.o -OBJS=rsync.o exclude.o util.o md4.o main.o checksum.o match.o flist.o $(LIBOBJ) +OBJS1=rsync.o exclude.o util.o md4.o main.o checksum.o match.o +OBJS=$(OBJS1) flist.o io.o $(LIBOBJ) .c.o: $(CC) $(CFLAGS) -c $*.c -o $*.o diff --git a/exclude.c b/exclude.c index b5b230b9..1c51607f 100644 --- a/exclude.c +++ b/exclude.c @@ -35,7 +35,6 @@ static int is_regex(char *str) static int check_one_exclude(char *name,char *pattern) { - char *str; char *p; if (!strchr(pattern,'/') && (p=strrchr(name,'/'))) diff --git a/io.c b/io.c new file mode 100644 index 00000000..5ca0e6c6 --- /dev/null +++ b/io.c @@ -0,0 +1,219 @@ +/* + Copyright (C) Andrew Tridgell 1996 + Copyright (C) Paul Mackerras 1996 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +/* + Utilities used in rsync + + tridge, June 1996 + */ +#include "rsync.h" + +static int total_written = 0; +static int total_read = 0; + +extern int verbose; + +int write_total(void) +{ + return total_written; +} + +int read_total(void) +{ + return total_read; +} + +static int buffer_f_in = -1; + +void setup_nonblocking(int f_in,int f_out) +{ + set_blocking(f_out,0); + buffer_f_in = f_in; +} + + +static char *read_buffer = NULL; +static char *read_buffer_p = NULL; +static int read_buffer_len = 0; +static int read_buffer_size = 0; + + +/* This function was added to overcome a deadlock problem when using + * ssh. It looks like we can't allow our receive queue to get full or + * ssh will clag up. Uggh. */ +static void read_check(int f) +{ + int n; + + if (read_buffer_len == 0) { + read_buffer_p = read_buffer; + } + + if ((n=num_waiting(f)) <= 0) + return; + + if (read_buffer_p != read_buffer) { + memmove(read_buffer,read_buffer_p,read_buffer_len); + read_buffer_p = read_buffer; + } + + if (n > (read_buffer_size - read_buffer_len)) { + read_buffer_size += n; + if (!read_buffer) + read_buffer = (char *)malloc(read_buffer_size); + else + read_buffer = (char *)realloc(read_buffer,read_buffer_size); + if (!read_buffer) out_of_memory("read check"); + read_buffer_p = read_buffer; + } + + n = read(f,read_buffer+read_buffer_len,n); + if (n > 0) { + read_buffer_len += n; + } +} + + +static int readfd(int fd,char *buffer,int N) +{ + int ret; + int total=0; + + while (total < N) + { + if (read_buffer_len > 0) { + ret = MIN(read_buffer_len,N-total); + memcpy(buffer+total,read_buffer_p,ret); + read_buffer_p += ret; + read_buffer_len -= ret; + } else { + ret = read(fd,buffer + total,N - total); + } + + if (ret <= 0) + return total; + total += ret; + } + return total; +} + + +int read_int(int f) +{ + char b[4]; + if (readfd(f,b,4) != 4) { + if (verbose > 1) + fprintf(stderr,"Error reading %d bytes : %s\n",4,strerror(errno)); + exit(1); + } + total_read += 4; + return IVAL(b,0); +} + +void read_buf(int f,char *buf,int len) +{ + if (readfd(f,buf,len) != len) { + if (verbose > 1) + fprintf(stderr,"Error reading %d bytes : %s\n",len,strerror(errno)); + exit(1); + } + total_read += len; +} + + +int read_write(int fd_in,int fd_out,int size) +{ + static char *buf=NULL; + static int bufsize = WRITE_BLOCK_SIZE; + int total=0; + + if (!buf) { + buf = (char *)malloc(bufsize); + if (!buf) out_of_memory("read_write"); + } + + while (total < size) { + int n = MIN(size-total,bufsize); + read_buf(fd_in,buf,n); + if (write(fd_out,buf,n) != n) + return total; + total += n; + } + return total; +} + + +static int writefd(int fd,char *buf,int len) +{ + int total = 0; + fd_set fds; + + if (buffer_f_in == -1) + return write(fd,buf,len); + + while (total < len) { + int ret = write(fd,buf+total,len-total); + + if (ret == 0) return total; + + if (ret == -1 && errno != EWOULDBLOCK) + return total?total:-1; + + if (ret == -1) { + read_check(buffer_f_in); + + FD_ZERO(&fds); + FD_SET(fd,&fds); + select(16,NULL,&fds,NULL,NULL); + } else { + total += ret; + } + } + + return total; +} + + + +void write_int(int f,int x) +{ + char b[4]; + SIVAL(b,0,x); + if (writefd(f,b,4) != 4) { + fprintf(stderr,"write_int failed : %s\n",strerror(errno)); + exit(1); + } + total_written += 4; +} + +void write_buf(int f,char *buf,int len) +{ + if (writefd(f,buf,len) != len) { + fprintf(stderr,"write_buf failed : %s\n",strerror(errno)); + exit(1); + } + total_written += len; +} + + +void write_flush(int f) +{ +} + + diff --git a/match.c b/match.c index 3b205bfb..1591fdc6 100644 --- a/match.c +++ b/match.c @@ -38,9 +38,6 @@ static int total_matches=0; static int total_data_transfer=0; -static int check_f_in; - - struct target { tag t; int i; @@ -100,7 +97,6 @@ static void matched(int f,struct sum_struct *s,char *buf,off_t len,int offset,in (int)offset,(int)last_match,i,(int)s->sums[i].len,n); if (n > 0) { - read_check(check_f_in); write_int(f,n); write_buf(f,buf+last_match,n); data_transfer += n; @@ -201,7 +197,7 @@ static void hash_search(int f,struct sum_struct *s,char *buf,off_t len) } -void match_sums(int f,struct sum_struct *s,char *buf,off_t len,int f_in) +void match_sums(int f,struct sum_struct *s,char *buf,off_t len) { last_match = 0; false_alarms = 0; @@ -209,8 +205,6 @@ void match_sums(int f,struct sum_struct *s,char *buf,off_t len,int f_in) matches=0; data_transfer=0; - check_f_in = f_in; - if (len > 0 && s->count>0) { build_hash_table(s); diff --git a/rsync.c b/rsync.c index d9b7e617..9f1d80ab 100644 --- a/rsync.c +++ b/rsync.c @@ -611,10 +611,10 @@ off_t send_files(struct file_list *flist,int f_out,int f_in) if (verbose > 2) fprintf(stderr,"send_files starting\n"); + setup_nonblocking(f_in,f_out); + while (1) { - read_check(f_in); - i = read_int(f_in); if (i == -1) break; @@ -680,7 +680,7 @@ off_t send_files(struct file_list *flist,int f_out,int f_in) if (!am_server && verbose) printf("%s\n",fname); - match_sums(f_out,s,buf,st.st_size,f_in); + match_sums(f_out,s,buf,st.st_size); write_flush(f_out); unmap_file(buf,st.st_size); diff --git a/rsync.h b/rsync.h index ae430468..fd24df5e 100644 --- a/rsync.h +++ b/rsync.h @@ -236,3 +236,7 @@ extern int errno; #endif #define SIGNAL_CAST (RETSIGTYPE (*)()) + +#ifndef EWOULDBLOCK +#define EWOULDBLOCK EAGAIN +#endif diff --git a/util.c b/util.c index 837e8c45..df60d0a0 100644 --- a/util.c +++ b/util.c @@ -24,144 +24,13 @@ */ #include "rsync.h" -static int total_written = 0; -static int total_read = 0; - -extern int verbose; - -int write_total(void) -{ - return total_written; -} - -int read_total(void) -{ - return total_read; -} - -void write_int(int f,int x) -{ - char b[4]; - SIVAL(b,0,x); - if (write(f,b,4) != 4) { - fprintf(stderr,"write_int failed : %s\n",strerror(errno)); - exit(1); - } - total_written += 4; -} - -void write_buf(int f,char *buf,int len) -{ - if (write(f,buf,len) != len) { - fprintf(stderr,"write_buf failed : %s\n",strerror(errno)); - exit(1); - } - total_written += len; -} - -static int num_waiting(int fd) +int num_waiting(int fd) { int len=0; -#ifdef FIONREAD ioctl(fd,FIONREAD,&len); -#endif return(len); } -void write_flush(int f) -{ -} - - -static char *read_buffer = NULL; -static char *read_buffer_p = NULL; -static int read_buffer_len = 0; -static int read_buffer_size = 0; - - -/* This function was added to overcome a deadlock problem when using - * ssh. It looks like we can't allow our receive queue to get full or - * ssh will clag up. Uggh. */ -void read_check(int f) -{ - int n; - - if (read_buffer_len == 0) { - read_buffer_p = read_buffer; - } - - if ((n=num_waiting(f)) <= 0) - return; - - if (read_buffer_p != read_buffer) { - memmove(read_buffer,read_buffer_p,read_buffer_len); - read_buffer_p = read_buffer; - } - - if (n > (read_buffer_size - read_buffer_len)) { - read_buffer_size += n; /* deliberately overdo it a bit */ - if (!read_buffer) - read_buffer = (char *)malloc(read_buffer_size); - else - read_buffer = (char *)realloc(read_buffer,read_buffer_size); - if (!read_buffer) out_of_memory("read check"); - read_buffer_p = read_buffer; - } - - n = read(f,read_buffer+read_buffer_len,n); - if (n > 0) { - read_buffer_len += n; - } -} - - -static int readfd(int fd,char *buffer,int N) -{ - int ret; - int total=0; - - while (total < N) - { - if (read_buffer_len > 0) { - ret = MIN(read_buffer_len,N-total); - memcpy(buffer+total,read_buffer_p,ret); - read_buffer_p += ret; - read_buffer_len -= ret; - } else { - ret = read(fd,buffer + total,N - total); - } - - if (ret <= 0) - return total; - total += ret; - } - return total; -} - - -int read_int(int f) -{ - char b[4]; - if (readfd(f,b,4) != 4) { - if (verbose > 1) - fprintf(stderr,"Error reading %d bytes : %s\n",4,strerror(errno)); - exit(1); - } - total_read += 4; - return IVAL(b,0); -} - -void read_buf(int f,char *buf,int len) -{ - if (readfd(f,buf,len) != len) { - if (verbose > 1) - fprintf(stderr,"Error reading %d bytes : %s\n",len,strerror(errno)); - exit(1); - } - total_read += len; -} - - char *map_file(int fd,off_t len) { char *ret = (char *)mmap(NULL,len,PROT_READ,MAP_SHARED,fd,0); @@ -175,28 +44,6 @@ void unmap_file(char *buf,off_t len) } -int read_write(int fd_in,int fd_out,int size) -{ - static char *buf=NULL; - static int bufsize = WRITE_BLOCK_SIZE; - int total=0; - - if (!buf) { - buf = (char *)malloc(bufsize); - if (!buf) out_of_memory("read_write"); - } - - while (total < size) { - int n = MIN(size-total,bufsize); - read_buf(fd_in,buf,n); - if (write(fd_out,buf,n) != n) - return total; - total += n; - } - return total; -} - - /* this is taken from CVS */ int piped_child(char **command,int *f_in,int *f_out) { @@ -286,3 +133,33 @@ int set_modtime(char *fname,time_t modtime) #endif } + + +/**************************************************************************** +Set a fd into blocking/nonblocking mode. Uses POSIX O_NONBLOCK if available, +else +if SYSV use O_NDELAY +if BSD use FNDELAY +****************************************************************************/ +int set_blocking(int fd, int set) +{ + int val; +#ifdef O_NONBLOCK +#define FLAG_TO_SET O_NONBLOCK +#else +#ifdef SYSV +#define FLAG_TO_SET O_NDELAY +#else /* BSD */ +#define FLAG_TO_SET FNDELAY +#endif +#endif + + if((val = fcntl(fd, F_GETFL, 0)) == -1) + return -1; + if(set) /* Turn blocking on - ie. clear nonblock flag */ + val &= ~FLAG_TO_SET; + else + val |= FLAG_TO_SET; + return fcntl( fd, F_SETFL, val); +#undef FLAG_TO_SET +}