From a800434a82af8dbd59da7c27b382c9897dd90150 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Mon, 1 Jun 1998 03:42:14 +0000 Subject: [PATCH] added --stats option for verbose stats on the file transfer --- flist.c | 18 ++- io.c | 24 +--- main.c | 58 +++++--- match.c | 15 +- options.c | 11 +- rsync.c | 407 +++++++++++++++++++++++++++++------------------------- rsync.h | 18 ++- 7 files changed, 312 insertions(+), 239 deletions(-) diff --git a/flist.c b/flist.c index 138894f7..7a0db274 100644 --- a/flist.c +++ b/flist.c @@ -21,12 +21,13 @@ #include "rsync.h" +extern struct stats stats; + extern int csum_length; extern int verbose; extern int am_server; extern int always_checksum; -extern int64 total_size; extern int cvs_exclude; @@ -421,7 +422,7 @@ static struct file_struct *make_file(char *fname) } if (!S_ISDIR(st.st_mode)) - total_size += st.st_size; + stats.total_size += st.st_size; return file; } @@ -525,12 +526,15 @@ struct file_list *send_file_list(int f,int argc,char *argv[]) char *p,*dir; char lastpath[MAXPATHLEN]=""; struct file_list *flist; + int64 start_write; if (verbose && recurse && !am_server && f != -1) { rprintf(FINFO,"building file list ... "); rflush(FINFO); } + start_write = stats.total_written; + flist = (struct file_list *)malloc(sizeof(flist[0])); if (!flist) out_of_memory("send_file_list"); @@ -651,6 +655,8 @@ struct file_list *send_file_list(int f,int argc,char *argv[]) if (f != -1) { io_end_buffering(f); + stats.flist_size = stats.total_written - start_write; + stats.num_files = flist->count; } if (verbose > 2) @@ -664,12 +670,15 @@ struct file_list *recv_file_list(int f) { struct file_list *flist; unsigned char flags; + int64 start_read; if (verbose && recurse && !am_server) { rprintf(FINFO,"receiving file list ... "); rflush(FINFO); } + start_read = stats.total_read; + flist = (struct file_list *)malloc(sizeof(flist[0])); if (!flist) goto oom; @@ -700,7 +709,7 @@ struct file_list *recv_file_list(int f) receive_file_entry(&flist->files[i],flags,f); if (S_ISREG(flist->files[i]->mode)) - total_size += flist->files[i]->length; + stats.total_size += flist->files[i]->length; flist->count++; @@ -731,6 +740,9 @@ struct file_list *recv_file_list(int f) if (verbose > 2) rprintf(FINFO,"recv_file_list done\n"); + stats.flist_size = stats.total_read - start_read; + stats.num_files = flist->count; + return flist; oom: diff --git a/io.c b/io.c index a88d76af..61bdc609 100644 --- a/io.c +++ b/io.c @@ -24,9 +24,6 @@ */ #include "rsync.h" -static int64 total_written; -static int64 total_read; - static int io_multiplexing_out; static int io_multiplexing_in; static int multiplex_in_fd; @@ -35,17 +32,7 @@ static time_t last_io; static int eof_error=1; extern int verbose; extern int io_timeout; - - -int64 write_total(void) -{ - return total_written; -} - -int64 read_total(void) -{ - return total_read; -} +extern struct stats stats; static int buffer_f_in = -1; @@ -108,6 +95,7 @@ static int read_timeout(int fd, char *buf, int len) n = read(fd, buf, len); if (n > 0) { + stats.total_read += n; buf += n; len -= n; ret += n; @@ -267,7 +255,6 @@ int32 read_int(int f) { char b[4]; readfd(f,b,4); - total_read += 4; return IVAL(b,0); } @@ -286,7 +273,6 @@ int64 read_longint(int f) #else if (remote_version >= 16) { readfd(f,b,8); - total_read += 8; ret = IVAL(b,0) | (((int64)IVAL(b,4))<<32); } #endif @@ -297,7 +283,6 @@ int64 read_longint(int f) void read_buf(int f,char *buf,int len) { readfd(f,buf,len); - total_read += len; } void read_sbuf(int f,char *buf,int len) @@ -368,6 +353,8 @@ static void writefd_unbuffered(int fd,char *buf,int len) } total += ret; + stats.total_written += ret; + if (io_timeout) last_io = time(NULL); continue; @@ -446,7 +433,6 @@ void write_int(int f,int32 x) char b[4]; SIVAL(b,0,x); writefd(f,b,4); - total_written += 4; } void write_longint(int f, int64 x) @@ -464,13 +450,11 @@ void write_longint(int f, int64 x) SIVAL(b,4,((x>>32)&0xFFFFFFFF)); writefd(f,b,8); - total_written += 8; } void write_buf(int f,char *buf,int len) { writefd(f,buf,len); - total_written += len; } /* write a string to the connection */ diff --git a/main.c b/main.c index fa4fe0f7..6fd96a7c 100644 --- a/main.c +++ b/main.c @@ -20,7 +20,8 @@ #include "rsync.h" time_t starttime = 0; -int64 total_size = 0; + +struct stats stats; extern int csum_length; @@ -28,16 +29,17 @@ extern int verbose; static void report(int f) { - int64 in,out,tsize; time_t t = time(NULL); extern int am_server; extern int am_sender; extern int am_daemon; + extern int do_stats; if (am_daemon) { syslog(LOG_INFO,"wrote %.0f bytes read %.0f bytes total size %.0f\n", - (double)write_total(),(double)read_total(), - (double)total_size); + (double)stats.total_written, + (double)stats.total_read, + (double)stats.total_size); if (f == -1 || !am_sender) return; } @@ -46,26 +48,46 @@ static void report(int f) if (am_server && !am_sender) return; if (am_server && am_sender) { - write_longint(f,read_total()); - write_longint(f,write_total()); - write_longint(f,total_size); + write_longint(f,stats.total_read); + write_longint(f,stats.total_written); + write_longint(f,stats.total_size); return; } - if (am_sender) { - in = read_total(); - out = write_total(); - tsize = total_size; - } else { - out = read_longint(f); - in = read_longint(f); - tsize = read_longint(f); + if (!am_sender) { + int64 r; + stats.total_written = read_longint(f); + r = read_longint(f); + stats.total_size = read_longint(f); + stats.total_read = r; + } + + if (do_stats) { + printf("\nNumber of files: %d\n", stats.num_files); + printf("Number of files transferred: %d\n", + stats.num_transferred_files); + printf("Total file size: %.0f bytes\n", + (double)stats.total_size); + printf("Total transferred file size: %.0f bytes\n", + (double)stats.total_transferred_size); + printf("Literal data: %.0f bytes\n", + (double)stats.literal_data); + printf("Matched data: %.0f bytes\n", + (double)stats.matched_data); + printf("File list size: %d\n", stats.flist_size); + printf("Total bytes written: %.0f\n", + (double)stats.total_written); + printf("Total bytes read: %.0f\n\n", + (double)stats.total_read); } printf("wrote %.0f bytes read %.0f bytes %.2f bytes/sec\n", - (double)out,(double)in,(in+out)/(0.5 + (t-starttime))); + (double)stats.total_written, + (double)stats.total_read, + (stats.total_written+stats.total_read)/(0.5 + (t-starttime))); printf("total size is %.0f speedup is %.2f\n", - (double)tsize,(1.0*tsize)/(in+out)); + (double)stats.total_size, + (1.0*stats.total_size)/(stats.total_written+stats.total_read)); } @@ -498,6 +520,8 @@ int main(int argc,char *argv[]) starttime = time(NULL); am_root = (getuid() == 0); + memset(&stats, 0, sizeof(stats)); + if (argc < 2) { usage(FERROR); exit_cleanup(1); diff --git a/match.c b/match.c index 333f0cf2..719961da 100644 --- a/match.c +++ b/match.c @@ -34,13 +34,13 @@ typedef unsigned short tag; static int false_alarms; static int tag_hits; static int matches; -static int data_transfer; +static int64 data_transfer; static int total_false_alarms; static int total_tag_hits; static int total_matches; -static int64 total_data_transfer; +extern struct stats stats; struct target { tag t; @@ -103,8 +103,10 @@ static void matched(int f,struct sum_struct *s,struct map_struct *buf, send_token(f,i,buf,last_match,n,i<0?0:s->sums[i].len); data_transfer += n; - if (i >= 0) + if (i >= 0) { + stats.matched_data += s->sums[i].len; n += s->sums[i].len; + } for (j=0;j 3) rprintf(FINFO,"data recv %d at %d\n",i,(int)offset); + stats.literal_data += i; sum_update(data,i); if (fd != -1 && write_file(fd,data,i) != i) { @@ -573,6 +575,8 @@ static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname) if (i == count-1 && remainder != 0) len = remainder; + stats.matched_data += len; + if (verbose > 3) rprintf(FINFO,"chunk[%d] of size %d at %d offset=%d\n", i,len,(int)offset2,(int)offset); @@ -739,215 +743,237 @@ void sig_int(void) } -int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) -{ - int fd1,fd2; - STRUCT_STAT st; - char *fname; - char fnametmp[MAXPATHLEN]; - struct map_struct *buf; - int i; - struct file_struct *file; - int phase=0; - int recv_ok; - if (verbose > 2) { - rprintf(FINFO,"recv_files(%d) starting\n",flist->count); - } - if (recurse && delete_mode && !local_name && flist->count>0) { - delete_files(flist); - } +static int get_tmpname(char *fnametmp, char *fname) +{ + char *f; + + /* open tmp file */ + if (tmpdir) { + f = strrchr(fname,'/'); + if (f == NULL) + f = fname; + else + f++; + if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) { + rprintf(FERROR,"filename too long\n"); + return 0; + } + slprintf(fnametmp,MAXPATHLEN-1, "%s/.%s.XXXXXX",tmpdir,f); + return 1; + } - while (1) - { - i = read_int(f_in); - if (i == -1) { - if (phase==0 && remote_version >= 13) { - phase++; - csum_length = SUM_LENGTH; - if (verbose > 2) - rprintf(FINFO,"recv_files phase=%d\n",phase); - write_int(f_gen,-1); - continue; + f = strrchr(fname,'/'); + + if (strlen(fname)+9 > MAXPATHLEN) { + rprintf(FERROR,"filename too long\n"); + return 0; } - break; - } - file = flist->files[i]; - fname = f_name(file); + if (f) { + *f = 0; + slprintf(fnametmp,MAXPATHLEN-1,"%s/.%s.XXXXXX", + fname,f+1); + *f = '/'; + } else { + slprintf(fnametmp,MAXPATHLEN-1,".%s.XXXXXX",fname); + } - if (local_name) - fname = local_name; + return 1; +} - if (dry_run) { - if (!am_server && verbose) - printf("%s\n",fname); - continue; - } +int recv_files(int f_in,struct file_list *flist,char *local_name,int f_gen) +{ + int fd1,fd2; + STRUCT_STAT st; + char *fname; + char fnametmp[MAXPATHLEN]; + struct map_struct *buf; + int i; + struct file_struct *file; + int phase=0; + int recv_ok; + + if (verbose > 2) { + rprintf(FINFO,"recv_files(%d) starting\n",flist->count); + } - if (verbose > 2) - rprintf(FINFO,"recv_files(%s)\n",fname); + if (recurse && delete_mode && !local_name && flist->count>0) { + delete_files(flist); + } - /* open the file */ - fd1 = open(fname,O_RDONLY); + while (1) { + i = read_int(f_in); + if (i == -1) { + if (phase==0 && remote_version >= 13) { + phase++; + csum_length = SUM_LENGTH; + if (verbose > 2) + rprintf(FINFO,"recv_files phase=%d\n",phase); + write_int(f_gen,-1); + continue; + } + break; + } - if (fd1 != -1 && do_fstat(fd1,&st) != 0) { - rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno)); - receive_data(f_in,NULL,-1,NULL); - close(fd1); - continue; - } + if (i < 0 || i >= flist->count) { + rprintf(FERROR,"Invalid file index %d in recv_files (count=%d)\n", + i, flist->count); + exit_cleanup(1); + } - if (fd1 != -1 && !S_ISREG(st.st_mode)) { - rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname); - receive_data(f_in,NULL,-1,NULL); - close(fd1); - continue; - } + file = flist->files[i]; + fname = f_name(file); - if (fd1 != -1 && st.st_size > 0) { - buf = map_file(fd1,st.st_size); - if (verbose > 2) - rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size); - } else { - buf = NULL; - } + stats.num_transferred_files++; + stats.total_transferred_size += file->length; - /* open tmp file */ - if (tmpdir) { - char *f; - f = strrchr(fname,'/'); - if (f == NULL) - f = fname; - else - f++; - if (strlen(tmpdir)+strlen(f)+10 > MAXPATHLEN) { - rprintf(FERROR,"filename too long\n"); - if (buf) unmap_file(buf); - close(fd1); - continue; - } - slprintf(fnametmp,sizeof(fnametmp)-1, "%s/.%s.XXXXXX",tmpdir,f); - } else { - char *f = strrchr(fname,'/'); - - if (strlen(fname)+9 > MAXPATHLEN) { - rprintf(FERROR,"filename too long\n"); - if (buf) unmap_file(buf); - close(fd1); - continue; - } - - if (f) { - *f = 0; - slprintf(fnametmp,sizeof(fnametmp)-1,"%s/.%s.XXXXXX",fname,f+1); - *f = '/'; - } else { - slprintf(fnametmp,sizeof(fnametmp)-1,".%s.XXXXXX",fname); - } - } - if (NULL == do_mktemp(fnametmp)) { - rprintf(FERROR,"mktemp %s failed\n",fnametmp); - receive_data(f_in,buf,-1,NULL); - if (buf) unmap_file(buf); - close(fd1); - continue; - } - fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode); - if (fd2 == -1 && relative_paths && errno == ENOENT && - create_directory_path(fnametmp) == 0) { - fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode); - } - if (fd2 == -1) { - rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno)); - receive_data(f_in,buf,-1,NULL); - if (buf) unmap_file(buf); - close(fd1); - continue; - } - - cleanup_fname = fnametmp; + if (local_name) + fname = local_name; - if (!am_server && verbose) - printf("%s\n",fname); + if (dry_run) { + if (!am_server && verbose) + printf("%s\n",fname); + continue; + } - /* recv file data */ - recv_ok = receive_data(f_in,buf,fd2,fname); + if (verbose > 2) + rprintf(FINFO,"recv_files(%s)\n",fname); - if (buf) unmap_file(buf); - if (fd1 != -1) { - close(fd1); - } - close(fd2); + /* open the file */ + fd1 = open(fname,O_RDONLY); - if (verbose > 2) - rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname); + if (fd1 != -1 && do_fstat(fd1,&st) != 0) { + rprintf(FERROR,"fstat %s : %s\n",fname,strerror(errno)); + receive_data(f_in,NULL,-1,NULL); + close(fd1); + continue; + } - if (make_backups) { - char fnamebak[MAXPATHLEN]; - if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) { - rprintf(FERROR,"backup filename too long\n"); - continue; - } - slprintf(fnamebak,sizeof(fnamebak)-1,"%s%s",fname,backup_suffix); - if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) { - rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno)); - continue; - } - } + if (fd1 != -1 && !S_ISREG(st.st_mode)) { + rprintf(FERROR,"%s : not a regular file (recv_files)\n",fname); + receive_data(f_in,NULL,-1,NULL); + close(fd1); + continue; + } - /* move tmp file over real file */ - if (do_rename(fnametmp,fname) != 0) { - if (errno == EXDEV) { - /* rename failed on cross-filesystem link. - Copy the file instead. */ - if (copy_file(fnametmp,fname, file->mode)) { - rprintf(FERROR,"copy %s -> %s : %s\n", - fnametmp,fname,strerror(errno)); - } else { - set_perms(fname,file,NULL,0); - } - do_unlink(fnametmp); - } else { - rprintf(FERROR,"rename %s -> %s : %s\n", - fnametmp,fname,strerror(errno)); - do_unlink(fnametmp); - } - } else { - set_perms(fname,file,NULL,0); - } + if (fd1 != -1 && st.st_size > 0) { + buf = map_file(fd1,st.st_size); + if (verbose > 2) + rprintf(FINFO,"recv mapped %s of size %d\n",fname,(int)st.st_size); + } else { + buf = NULL; + } - cleanup_fname = NULL; + if (!get_tmpname(fnametmp,fname)) { + if (buf) unmap_file(buf); + close(fd1); + continue; + } + if (NULL == do_mktemp(fnametmp)) { + rprintf(FERROR,"mktemp %s failed\n",fnametmp); + receive_data(f_in,buf,-1,NULL); + if (buf) unmap_file(buf); + close(fd1); + continue; + } - if (!recv_ok) { - if (csum_length == SUM_LENGTH) { - rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n", - fname); - } else { - if (verbose > 1) - rprintf(FINFO,"redoing %s(%d)\n",fname,i); - write_int(f_gen,i); - } - } - } + fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode); + if (fd2 == -1 && relative_paths && errno == ENOENT && + create_directory_path(fnametmp) == 0) { + fd2 = do_open(fnametmp,O_WRONLY|O_CREAT|O_EXCL,file->mode); + } + if (fd2 == -1) { + rprintf(FERROR,"open %s : %s\n",fnametmp,strerror(errno)); + receive_data(f_in,buf,-1,NULL); + if (buf) unmap_file(buf); + close(fd1); + continue; + } + + cleanup_fname = fnametmp; + + if (!am_server && verbose) + printf("%s\n",fname); + + /* recv file data */ + recv_ok = receive_data(f_in,buf,fd2,fname); + + if (buf) unmap_file(buf); + if (fd1 != -1) { + close(fd1); + } + close(fd2); + + if (verbose > 2) + rprintf(FINFO,"renaming %s to %s\n",fnametmp,fname); + + if (make_backups) { + char fnamebak[MAXPATHLEN]; + if (strlen(fname) + strlen(backup_suffix) > (MAXPATHLEN-1)) { + rprintf(FERROR,"backup filename too long\n"); + continue; + } + slprintf(fnamebak,sizeof(fnamebak)-1,"%s%s",fname,backup_suffix); + if (do_rename(fname,fnamebak) != 0 && errno != ENOENT) { + rprintf(FERROR,"rename %s %s : %s\n",fname,fnamebak,strerror(errno)); + continue; + } + } - if (preserve_hard_links) - do_hard_links(flist); + /* move tmp file over real file */ + if (do_rename(fnametmp,fname) != 0) { + if (errno == EXDEV) { + /* rename failed on cross-filesystem link. + Copy the file instead. */ + if (copy_file(fnametmp,fname, file->mode)) { + rprintf(FERROR,"copy %s -> %s : %s\n", + fnametmp,fname,strerror(errno)); + } else { + set_perms(fname,file,NULL,0); + } + do_unlink(fnametmp); + } else { + rprintf(FERROR,"rename %s -> %s : %s\n", + fnametmp,fname,strerror(errno)); + do_unlink(fnametmp); + } + } else { + set_perms(fname,file,NULL,0); + } - /* now we need to fix any directory permissions that were - modified during the transfer */ - for (i = 0; i < flist->count; i++) { - file = flist->files[i]; - if (!file->basename || !S_ISDIR(file->mode)) continue; - recv_generator(f_name(file),flist,i,-1); - } + cleanup_fname = NULL; + + + if (!recv_ok) { + if (csum_length == SUM_LENGTH) { + rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n", + fname); + } else { + if (verbose > 1) + rprintf(FINFO,"redoing %s(%d)\n",fname,i); + write_int(f_gen,i); + } + } + } - if (verbose > 2) - rprintf(FINFO,"recv_files finished\n"); - - return 0; + if (preserve_hard_links) + do_hard_links(flist); + + /* now we need to fix any directory permissions that were + modified during the transfer */ + for (i = 0; i < flist->count; i++) { + file = flist->files[i]; + if (!file->basename || !S_ISDIR(file->mode)) continue; + recv_generator(f_name(file),flist,i,-1); + } + + if (verbose > 2) + rprintf(FINFO,"recv_files finished\n"); + + return 0; } @@ -984,8 +1010,17 @@ void send_files(struct file_list *flist,int f_out,int f_in) break; } + if (i < 0 || i >= flist->count) { + rprintf(FERROR,"Invalid file index %d (count=%d)\n", + i, flist->count); + exit_cleanup(1); + } + file = flist->files[i]; + stats.num_transferred_files++; + stats.total_transferred_size += file->length; + fname[0] = 0; if (file->basedir) { strlcpy(fname,file->basedir,MAXPATHLEN-1); @@ -1133,10 +1168,6 @@ void generate_files(int f,struct file_list *flist,char *local_name,int f_recv) write_int(f,-1); } - - - if (verbose > 2) - rprintf(FINFO,"generator wrote %ld\n",(long)write_total()); } diff --git a/rsync.h b/rsync.h index 2964dc10..41f480ec 100644 --- a/rsync.h +++ b/rsync.h @@ -280,9 +280,9 @@ struct file_struct { }; struct file_list { - int count; - int malloced; - struct file_struct **files; + int count; + int malloced; + struct file_struct **files; }; struct sum_buf { @@ -316,6 +316,18 @@ struct exclude_struct { int local; }; +struct stats { + int64 total_size; + int64 total_transferred_size; + int64 total_written; + int64 total_read; + int64 literal_data; + int64 matched_data; + int flist_size; + int num_files; + int num_transferred_files; +}; + /* we need this function because of the silly way in which duplicate entries are handled in the file lists - we can't change this -- 2.34.1