From 6902ed178ba54c8e1c410519f7ee576ddb42554c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Aug 2001 02:04:47 +0000 Subject: [PATCH] Merge across rsync+ patch; add a little documentation to the manpage. More documentation would be better. --- Makefile.in | 2 +- NEWS | 4 + batch.c | 572 ++++++++++++++++++++++++++++++++++++++++++++++++++++ compat.c | 6 + flist.c | 10 +- main.c | 58 +++++- match.c | 3 + options.c | 23 ++- rsync.c | 1 + rsync.yo | 32 +++ sender.c | 88 +++++++- token.c | 33 +++ util.c | 6 +- 13 files changed, 815 insertions(+), 23 deletions(-) create mode 100644 batch.c diff --git a/Makefile.in b/Makefile.in index 3bf7d492..56e1b042 100644 --- a/Makefile.in +++ b/Makefile.in @@ -26,7 +26,7 @@ ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \ zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \ zlib/zutil.o zlib/adler32.o OBJS1=rsync.o generator.o receiver.o cleanup.o sender.o exclude.o util.o main.o checksum.o match.o syscall.o log.o backup.o -OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o fileio.o +OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o fileio.o batch.o DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ popt/popthelp.o popt/poptparse.o diff --git a/NEWS b/NEWS index a5ce2542..c9a4c71d 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,10 @@ rsync 2.4.7 (sometime in 2001, maybe :) * Shell wildcards are allowed in "auth users" lines. + * Merged UNC rsync+ patch to support creation of standalone patch + sets. By Bert J. Dempsey and Debra Weiss, updated by Jos + Backus. + ENHANCEMENTS * Include/exclude cluestick: with -vv, print out whether files are diff --git a/batch.c b/batch.c new file mode 100644 index 00000000..200ff8bc --- /dev/null +++ b/batch.c @@ -0,0 +1,572 @@ +/* + Weiss 1/1999 + Batch utilities + +*/ + +#include "rsync.h" +#include + +char rsync_flist_file[27] = "rsync_flist."; +char rsync_csums_file[27] = "rsync_csums."; +char rsync_delta_file[27] = "rsync_delta."; +char rsync_argvs_file[27] = "rsync_argvs."; + +char batch_file_ext[15]; + +int fdb; +int fdb_delta; +int fdb_open; +int fdb_close; + +struct file_list *batch_flist; + +void create_batch_file_ext() +{ + struct tm *timeptr; + time_t elapsed_seconds; + + /* Save run date and time to use for batch file extensions */ + time(&elapsed_seconds); + timeptr = localtime(&elapsed_seconds); + + sprintf(batch_file_ext, "%4d%02d%02d%02d%02d%02d", + timeptr->tm_year+1900, timeptr->tm_mon+1, timeptr->tm_mday, + timeptr->tm_hour, timeptr->tm_min, timeptr->tm_sec); +} + +void set_batch_file_ext(char *ext) +{ + strcpy(batch_file_ext, ext); +} + +void write_batch_flist_file(char *buff, int bytes_to_write) +{ + + if (fdb_open) { + /* Set up file extension */ + strcat(rsync_flist_file, batch_file_ext); + + /* Open batch flist file for writing; create it if it doesn't exist */ + fdb = do_open(rsync_flist_file, O_WRONLY|O_CREAT|O_TRUNC, + S_IREAD|S_IWRITE); + if (fdb == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", + rsync_flist_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } + fdb_open = 0; + } + + /* Write buffer to batch flist file */ + + if ( write(fdb, buff, bytes_to_write) == -1 ) { + rprintf(FERROR, "Batch file %s write error: %s\n", + rsync_flist_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } + + + if (fdb_close) { + close(fdb); + } +} + +void write_batch_flist_info(int flist_count, struct file_struct **fptr) +{ + int i; + int bytes_to_write; + + /* Write flist info to batch file */ + + bytes_to_write = sizeof(unsigned) + + sizeof(time_t) + + sizeof(OFF_T) + + sizeof(mode_t) + + sizeof(INO_T) + + (2 * sizeof(dev_t)) + + sizeof(uid_t) + + sizeof(gid_t); + + fdb_open = 1; + fdb_close = 0; + + for (i=0; ibasename); + write_char_bufs(fptr[i]->dirname); + write_char_bufs(fptr[i]->basedir); + write_char_bufs(fptr[i]->link); + if (i==flist_count - 1) { + fdb_close = 1; + } + write_char_bufs(fptr[i]->sum); + } + +} + +void write_char_bufs(char *buf) +{ + /* Write the size of the string which will follow */ + + char b[4]; + if (buf != NULL) + SIVAL(b,0,strlen(buf)); + else { + SIVAL(b,0,0); + } + + write_batch_flist_file(b, sizeof(int)); + + /* Write the string if there is one */ + + if (buf != NULL) { + write_batch_flist_file(buf, strlen(buf)); + } +} + +void write_batch_argvs_file(int orig_argc, int argc, char **argv) +{ + int fdb; + int i; + char buff[256]; + + strcat(rsync_argvs_file, batch_file_ext); + + + /* Open batch argvs file for writing; create it if it doesn't exist */ + fdb = do_open(rsync_argvs_file, O_WRONLY|O_CREAT|O_TRUNC, + S_IREAD|S_IWRITE|S_IEXEC); + if (fdb == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", rsync_argvs_file, + strerror(errno)); + close(fdb); + exit_cleanup(1); + } + buff[0] = '\0'; + /* Write argvs info to batch file */ + + for (i=argc - orig_argc;icount=0; + batch_flist->malloced=1000; + batch_flist->files = (struct file_struct **)malloc(sizeof(batch_flist->files[0])* batch_flist->malloced); + if (!batch_flist->files) { + out_of_memory("create_flist_from_batch"); /* dw -- will exit */ + } + + for ( flags=read_batch_flags() ; flags; flags=read_batch_flags() ) { + + int i = batch_flist->count; + + if (i >= batch_flist->malloced) { + if (batch_flist->malloced < 1000) + batch_flist->malloced += 1000; + else + batch_flist->malloced *= 2; + batch_flist->files =(struct file_struct **)realloc(batch_flist->files, + sizeof(batch_flist->files[0])* + batch_flist->malloced); + if (!batch_flist->files) + out_of_memory("create_flist_from_batch"); + } + read_batch_flist_info(&batch_flist->files[i]); + batch_flist->files[i]->flags = flags; + + batch_flist->count++; + } + + return batch_flist; + +} + +int read_batch_flist_file(char *buff, int len) +{ + int bytes_read; + + if (fdb_open) { + + /* Set up file extension */ + strcat(rsync_flist_file, batch_file_ext); + + /* Open batch flist file for reading */ + fdb = do_open(rsync_flist_file, O_RDONLY, 0); + if (fdb == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", rsync_flist_file, + strerror(errno)); + close(fdb); + exit_cleanup(1); + } + fdb_open = 0; + } + + /* Read flist batch file */ + + bytes_read = read(fdb, buff, len); + + if (bytes_read == -1) { + rprintf(FERROR, "Batch file %s read error: %s\n", + rsync_flist_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } + if (bytes_read == 0) { /* EOF */ + close(fdb); + } + return bytes_read; +} + +unsigned char read_batch_flags() +{ + int flags; + + if (read_batch_flist_file((char *)&flags, 4) ) { + return 1; + } + else { + return 0; + } +} + +void read_batch_flist_info(struct file_struct **fptr) +{ + int int_str_len; + char char_str_len[4]; + char buff[256]; + struct file_struct *file; + + file = (struct file_struct *)malloc(sizeof(*file)); + if (!file) out_of_memory("read_batch_flist_info"); + memset((char *)file, 0, sizeof(*file)); + + (*fptr) = file; + + read_batch_flist_file((char *)&file->modtime, sizeof(time_t)); + read_batch_flist_file((char *)&file->length, sizeof(OFF_T)); + read_batch_flist_file((char *)&file->mode, sizeof(mode_t)); + read_batch_flist_file((char *)&file->inode, sizeof(INO_T)); + read_batch_flist_file((char *)&file->dev, sizeof(dev_t)); + read_batch_flist_file((char *)&file->rdev, sizeof(dev_t)); + read_batch_flist_file((char *)&file->uid, sizeof(uid_t)); + read_batch_flist_file((char *)&file->gid, sizeof(gid_t)); + read_batch_flist_file(char_str_len, sizeof(char_str_len)); + int_str_len = IVAL(char_str_len,0); + if (int_str_len > 0) { + read_batch_flist_file(buff, int_str_len); + buff[int_str_len] = '\0'; + file->basename = strdup(buff); + } + else { + file->basename = NULL; + } + + read_batch_flist_file(char_str_len, sizeof(char_str_len)); + int_str_len = IVAL(char_str_len,0); + if (int_str_len > 0) { + read_batch_flist_file(buff, int_str_len); + buff[int_str_len] = '\0'; + file[0].dirname = strdup(buff); + } + else { + file[0].dirname = NULL; + } + + read_batch_flist_file(char_str_len, sizeof(char_str_len)); + int_str_len = IVAL(char_str_len,0); + if (int_str_len > 0) { + read_batch_flist_file(buff, int_str_len); + buff[int_str_len] = '\0'; + file[0].basedir = strdup(buff); + } + else { + file[0].basedir = NULL; + } + + read_batch_flist_file(char_str_len, sizeof(char_str_len)); + int_str_len = IVAL(char_str_len,0); + if (int_str_len > 0) { + read_batch_flist_file(buff, int_str_len); + buff[int_str_len] = '\0'; + file[0].link = strdup(buff); + } + else { + file[0].link = NULL; + } + + read_batch_flist_file(char_str_len, sizeof(char_str_len)); + int_str_len = IVAL(char_str_len,0); + if (int_str_len > 0) { + read_batch_flist_file(buff, int_str_len); + buff[int_str_len] = '\0'; + file[0].sum = strdup(buff); + } + else { + file[0].sum = NULL; + } +} + +void write_batch_csums_file(char *buff, int bytes_to_write) +{ + + static int fdb_open = 1; + + if (fdb_open) { + /* Set up file extension */ + strcat(rsync_csums_file, batch_file_ext); + + /* Open batch csums file for writing; create it if it doesn't exist */ + fdb = do_open(rsync_csums_file, O_WRONLY|O_CREAT|O_TRUNC, + S_IREAD|S_IWRITE); + if (fdb == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", + rsync_csums_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } + fdb_open = 0; + } + + /* Write buffer to batch csums file */ + + if ( write(fdb, buff, bytes_to_write) == -1 ) { + rprintf(FERROR, "Batch file %s write error: %s\n", + rsync_csums_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } +} + +void close_batch_csums_file() +{ + close(fdb); + +} + +void write_batch_csum_info(int *flist_entry, int flist_count, struct sum_struct *s) +{ + int i; + int int_zero = 0; + extern int block_size; + extern int csum_length; + + fdb_open = 1; + + /* Write csum info to batch file */ + + write_batch_csums_file ( (char *) flist_entry, sizeof(int) ); + write_batch_csums_file ( (char *) (s?&s->count:&int_zero), sizeof(int) ); + if (s) { + for (i=0; i < s->count; i++) { + write_batch_csums_file( (char *) &s->sums[i].sum1, sizeof(uint32)); + if ( (*flist_entry == flist_count - 1) && (i == s->count - 1) ) { + fdb_close = 1; + } + write_batch_csums_file( s->sums[i].sum2, csum_length); + } + } +} + +int read_batch_csums_file(char *buff, int len) +{ + static int fdb_open = 1; + int bytes_read; + + if (fdb_open) { + + /* Set up file extension */ + strcat(rsync_csums_file, batch_file_ext); + + /* Open batch flist file for reading */ + fdb = do_open(rsync_csums_file, O_RDONLY, 0); + if (fdb == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", rsync_csums_file, + strerror(errno)); + close(fdb); + exit_cleanup(1); + } + fdb_open = 0; + } + + /* Read csums batch file */ + + bytes_read = read(fdb, buff, len); + + if (bytes_read == -1) { + rprintf(FERROR, "Batch file %s read error: %s\n", + rsync_csums_file, strerror(errno)); + close(fdb); + exit_cleanup(1); + } + return bytes_read; +} + + +void read_batch_csum_info(int flist_entry, struct sum_struct *s, int *checksums_match) +{ + int i; + int file_flist_entry; + int file_chunk_ct; + uint32 file_sum1; + char file_sum2[SUM_LENGTH]; + extern int csum_length; + + + read_batch_csums_file((char *)&file_flist_entry, sizeof(int)); + if (file_flist_entry != flist_entry) { + rprintf(FINFO,"file_list_entry NE flist_entry\n"); + rprintf(FINFO,"file_flist_entry = %d flist_entry = %d\n", file_flist_entry, flist_entry); + close(fdb); + exit_cleanup(1); + + } + else { + read_batch_csums_file((char *)&file_chunk_ct, sizeof(int)); + *checksums_match = 1; + for (i = 0;i < file_chunk_ct;i++) { + + read_batch_csums_file((char *)&file_sum1, sizeof(uint32)); + read_batch_csums_file(file_sum2, csum_length); + + if ( (s->sums[i].sum1 != file_sum1) || + ( memcmp(s->sums[i].sum2,file_sum2, csum_length)!=0) ) { + *checksums_match = 0; + } + } /* end for */ + } + +} + +void write_batch_delta_file(char *buff, int bytes_to_write) +{ + static int fdb_delta_open = 1; + + if (fdb_delta_open) { + /* Set up file extension */ + strcat(rsync_delta_file, batch_file_ext); + + /* Open batch delta file for writing; create it if it doesn't exist */ + fdb_delta = do_open(rsync_delta_file, O_WRONLY|O_CREAT|O_TRUNC, + S_IREAD|S_IWRITE); + if (fdb_delta == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", + rsync_delta_file, strerror(errno)); + close(fdb_delta); + exit_cleanup(1); + } + fdb_delta_open = 0; + } + + /* Write buffer to batch delta file */ + + if ( write(fdb_delta, buff, bytes_to_write) == -1 ) { + rprintf(FERROR, "Batch file %s write error: %s\n", + rsync_delta_file, strerror(errno)); + close(fdb_delta); + exit_cleanup(1); + } +} +void close_batch_delta_file() +{ + close(fdb_delta); + +} + +int read_batch_delta_file(char *buff, int len) +{ + static int fdb_delta_open = 1; + int bytes_read; + + if (fdb_delta_open) { + + /* Set up file extension */ + strcat(rsync_delta_file, batch_file_ext); + + /* Open batch flist file for reading */ + fdb_delta = do_open(rsync_delta_file, O_RDONLY, 0); + if (fdb_delta == -1) { + rprintf(FERROR, "Batch file %s open error: %s\n", rsync_delta_file, + strerror(errno)); + close(fdb_delta); + exit_cleanup(1); + } + fdb_delta_open = 0; + } + + /* Read delta batch file */ + + bytes_read = read(fdb_delta, buff, len); + + if (bytes_read == -1) { + rprintf(FERROR, "Batch file %s read error: %s\n", + rsync_delta_file, strerror(errno)); + close(fdb_delta); + exit_cleanup(1); + } + return bytes_read; +} + + +void show_flist(int index, struct file_struct **fptr) +{ + /* for debugging show_flist(flist->count, flist->files **/ + + int i; + for (i=0;iflags=%x\n",fptr[i]->flags); + rprintf(FINFO,"flist->modtime=%x\n",fptr[i]->modtime); + rprintf(FINFO,"flist->length=%x\n",fptr[i]->length); + rprintf(FINFO,"flist->mode=%x\n",fptr[i]->mode); + rprintf(FINFO,"flist->basename=%s\n",fptr[i]->basename); + if (fptr[i]->dirname) + rprintf(FINFO,"flist->dirname=%s\n",fptr[i]->dirname); + if (fptr[i]->basedir) + rprintf(FINFO,"flist->basedir=%s\n",fptr[i]->basedir); + } +} + +void show_argvs(int argc, char *argv[]) +{ + /* for debugging **/ + + int i; + rprintf(FINFO,"BATCH.C:show_argvs,argc=%d\n",argc); + for (i=0;i= 12) { if (am_server) { + if (read_batch || write_batch) /* dw */ + checksum_seed = 32761; + else checksum_seed = time(NULL); write_int(f_out,checksum_seed); } else { diff --git a/flist.c b/flist.c index bb6149e7..edd6024f 100644 --- a/flist.c +++ b/flist.c @@ -47,6 +47,9 @@ extern int remote_version; extern int io_error; extern int sanitize_paths; +extern int read_batch; +extern int write_batch; + static char topsrcname[MAXPATHLEN]; static struct exclude_struct **local_exclude_list; @@ -613,6 +616,9 @@ void send_file_name(int f,struct file_list *flist,char *fname, out_of_memory("send_file_name"); } + if (write_batch) /* dw */ + file->flags = FLAG_DELETE; + if (strcmp(file->basename,"")) { flist->files[flist->count++] = file; send_file_entry(file,f,base_flags); @@ -841,6 +847,8 @@ struct file_list *send_file_list(int f,int argc,char *argv[]) io_end_buffering(f); stats.flist_size = stats.total_written - start_write; stats.num_files = flist->count; + if (write_batch) /* dw */ + write_batch_flist_info(flist->count, flist->files); } if (verbose > 2) @@ -918,7 +926,7 @@ struct file_list *recv_file_list(int f) } /* if protocol version is >= 17 then recv the io_error flag */ - if (f != -1 && remote_version >= 17) { + if (f != -1 && remote_version >= 17 && !read_batch) { /* dw-added readbatch */ extern int module_id; extern int ignore_errors; if (lp_ignore_errors(module_id) || ignore_errors) { diff --git a/main.c b/main.c index bc51571f..ec393527 100644 --- a/main.c +++ b/main.c @@ -135,8 +135,9 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int extern int local_server; extern char *rsync_path; extern int blocking_io; + extern int read_batch; - if (!local_server) { + if (!read_batch && !local_server) { /* dw -- added read_batch */ if (!cmd) cmd = getenv(RSYNC_RSH_ENV); if (!cmd) @@ -187,6 +188,8 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int } if (local_server) { + if (read_batch) + create_flist_from_batch(); ret = local_child(argc, args, f_in, f_out); } else { ret = piped_child(args,f_in,f_out); @@ -399,6 +402,9 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) extern int am_daemon; extern int module_id; extern int am_sender; + extern int read_batch; /* dw */ + extern int write_batch; /* dw */ + extern struct file_list *batch_flist; /* dw */ if (verbose > 2) rprintf(FINFO,"server_recv(%d) starting pid=%d\n",argc,(int)getpid()); @@ -424,7 +430,10 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) if (delete_mode && !delete_excluded) recv_exclude_list(f_in); - flist = recv_file_list(f_in); + if (read_batch) /* dw */ + flist = batch_flist; + else + flist = recv_file_list(f_in); if (!flist) { rprintf(FERROR,"server_recv: recv_file_list error\n"); exit_cleanup(RERR_FILESELECT); @@ -448,6 +457,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[]) extern int cvs_exclude; extern int am_sender; extern int remote_version; + extern int read_batch; /* dw */ setup_protocol(f_out, f_in); @@ -458,9 +468,11 @@ void start_server(int f_in, int f_out, int argc, char *argv[]) io_start_multiplex_out(f_out); if (am_sender) { - recv_exclude_list(f_in); - if (cvs_exclude) + if (!read_batch) { /* dw */ + recv_exclude_list(f_in); + if (cvs_exclude) add_cvs_excludes(); + } do_server_sender(f_in, f_out, argc, argv); } else { do_server_recv(f_in, f_out, argc, argv); @@ -481,8 +493,13 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) extern int am_sender; extern int remote_version; extern pid_t cleanup_child_pid; + extern int write_batch; /* dw */ + extern int read_batch; /* dw */ + extern struct file_list *batch_flist; /* dw */ cleanup_child_pid = pid; + if (read_batch) + flist = batch_flist; /* dw */ set_nonblocking(f_in); set_nonblocking(f_out); @@ -500,7 +517,8 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) add_cvs_excludes(); if (delete_mode && !delete_excluded) send_exclude_list(f_out); - flist = send_file_list(f_out,argc,argv); + if (!read_batch) /* dw -- don't write to pipe */ + flist = send_file_list(f_out,argc,argv); if (verbose > 3) rprintf(FINFO,"file list sent\n"); @@ -524,7 +542,8 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) list_only = 1; } - send_exclude_list(f_out); + if (!write_batch) /* dw */ + send_exclude_list(f_out); flist = recv_file_list(f_in); if (!flist || flist->count == 0) { @@ -584,6 +603,7 @@ static int start_client(int argc, char *argv[]) extern int rsync_port; extern int whole_file; char *argv0 = strdup(argv[0]); + extern int read_batch; if (strncasecmp(URL_PREFIX, argv0, strlen(URL_PREFIX)) == 0) { char *host, *path; @@ -604,7 +624,8 @@ static int start_client(int argc, char *argv[]) return start_socket_client(host, path, argc-1, argv+1); } - p = find_colon(argv0); + if (!read_batch) { /* dw */ + p = find_colon(argv[0]); if (p) { if (p[1] == ':') { @@ -651,7 +672,12 @@ static int start_client(int argc, char *argv[]) } argc--; } - + } else { + am_sender = 1; /* dw */ + local_server = 1; /* dw */ + shell_path = argv[argc-1]; /* dw */ + } + if (shell_machine) { p = strchr(shell_machine,'@'); if (p) { @@ -714,6 +740,13 @@ int main(int argc,char *argv[]) extern int am_daemon; extern int am_server; int ret; + extern int read_batch; /* dw */ + extern int write_batch; /* dw */ + extern char *batch_ext; /* dw */ + int i; /* dw */ + int orig_argc; /* dw */ + + orig_argc = argc; /* dw */ signal(SIGUSR1, sigusr1_handler); signal(SIGUSR2, sigusr2_handler); @@ -751,6 +784,15 @@ int main(int argc,char *argv[]) that implement getcwd that way "pwd" can't be found after chroot. */ push_dir(NULL,0); + if (write_batch) { /* dw */ + create_batch_file_ext(); + write_batch_argvs_file(orig_argc, argc, argv); + } + + if (read_batch) { /* dw */ + set_batch_file_ext(batch_ext); + } + if (am_daemon) { return daemon_main(); } diff --git a/match.c b/match.c index c0ae38c8..7b0f6017 100644 --- a/match.c +++ b/match.c @@ -260,6 +260,7 @@ static void hash_search(int f,struct sum_struct *s, void match_sums(int f,struct sum_struct *s,struct map_struct *buf,OFF_T len) { char file_sum[MD4_SUM_LENGTH]; + extern int write_batch; /* dw */ last_match = 0; false_alarms = 0; @@ -295,6 +296,8 @@ void match_sums(int f,struct sum_struct *s,struct map_struct *buf,OFF_T len) if (verbose > 2) rprintf(FINFO,"sending file_sum\n"); write_buf(f,file_sum,MD4_SUM_LENGTH); + if (write_batch) /* dw */ + write_batch_delta_file(file_sum, MD4_SUM_LENGTH); } if (targets) { diff --git a/options.c b/options.c index 35a6dc64..bfce8f68 100644 --- a/options.c +++ b/options.c @@ -74,6 +74,9 @@ int modify_window=0; #endif int blocking_io=0; +int read_batch=0; /* dw */ +int write_batch=0; /* dw */ + char *backup_suffix = BACKUP_SUFFIX; char *tmpdir = NULL; char *compare_dest = NULL; @@ -90,6 +93,8 @@ int quiet = 0; int always_checksum = 0; int list_only = 0; +char *batch_ext = NULL; + static int modify_window_set; @@ -206,6 +211,8 @@ void usage(enum logcode F) rprintf(F," --log-format=FORMAT log file transfers using specified format\n"); rprintf(F," --password-file=FILE get password from FILE\n"); rprintf(F," --bwlimit=KBPS limit I/O bandwidth, KBytes per second\n"); + rprintf(F," -f --read-batch=FILE read batch file\n"); + rprintf(F," -F --write-batch write batch file\n"); rprintf(F," -h, --help show this help screen\n"); rprintf(F,"\n"); @@ -290,10 +297,11 @@ static struct poptOption long_options[] = { {"address", 0, POPT_ARG_STRING, 0, OPT_ADDRESS}, {"backup-dir", 0, POPT_ARG_STRING, &backup_dir}, {"hard-links", 'H', POPT_ARG_NONE, &preserve_hard_links}, + {"read-batch", 'f', POPT_ARG_STRING, &batch_ext, 'f'}, + {"write-batch", 'F', POPT_ARG_NONE, &write_batch, 0}, {0,0,0,0} }; - static char err_buf[100]; @@ -468,6 +476,11 @@ int parse_arguments(int *argc, const char ***argv, int frommain) } break; + case 'f': + /* The filename is stored for us by popt */ + read_batch = 1; + break; + default: /* FIXME: If --daemon is specified, then errors for later * parameters seem to disappear. */ @@ -501,6 +514,7 @@ void server_options(char **args,int *argc) static char mdelete[30]; static char mwindow[30]; static char bw[50]; + static char fext[20]; /* dw */ int i, x; @@ -555,6 +569,8 @@ void server_options(char **args,int *argc) argstr[x++] = 'S'; if (do_compression) argstr[x++] = 'z'; + if (write_batch) + argstr[x++] = 'F'; /* dw */ /* this is a complete hack - blame Rusty @@ -576,6 +592,11 @@ void server_options(char **args,int *argc) snprintf(mdelete,sizeof(mdelete),"--max-delete=%d",max_delete); args[ac++] = mdelete; } + + if (batch_ext != NULL) { + sprintf(fext,"-f%s",batch_ext); + args[ac++] = fext; + } if (io_timeout) { snprintf(iotime,sizeof(iotime),"--timeout=%d",io_timeout); diff --git a/rsync.c b/rsync.c index 3e50f38d..890d6a89 100644 --- a/rsync.c +++ b/rsync.c @@ -226,6 +226,7 @@ int set_perms(char *fname,struct file_struct *file,STRUCT_STAT *st, void sig_int(void) { + rprintf(FINFO,"\nrsync.c:sig_int() called.\n"); exit_cleanup(RERR_SIGNAL); } diff --git a/rsync.yo b/rsync.yo index 3717b9e5..d4209ab4 100644 --- a/rsync.yo +++ b/rsync.yo @@ -276,7 +276,11 @@ verb( --log-format=FORMAT log file transfers using specified format --password-file=FILE get password from FILE --bwlimit=KBPS limit I/O bandwidth, KBytes per second + -f, --read-batch=FILE read batch file + -F, --write-batch write batch file -h, --help show this help screen + + ) manpageoptions() @@ -675,6 +679,11 @@ transfer was too fast, it will wait before sending the next data block. The result is an average transfer rate equalling the specified limit. A value of zero specifies no limit. +dit(bf(--read-batch)) Apply a previously generated change batch. + +dit(bf(--write-batch)) Generate a set of files that can be transferred +as a batch update. + enddit() manpagesection(EXCLUDE PATTERNS) @@ -765,6 +774,29 @@ itemize( it would be excluded by the "*") ) +manpagesection(BATCH MODE) + +The following call generates 4 files that encapsulate the information +for synchronizing the contents of bf(target_dir) with the updates found in +bf(src_dir) + +quote( +$ rsync -F [other rsync options here] \nl() + /somewhere/src_dir /somewhere/target_dir +) + +The generated files are labeled with a common timestamp: + +itemize( +it() bf(rsync_argvs.) command-line arguments +it() bf(rsync_flist.) rsync internal file metadata +it() bf(rsync_csums.) rsync checksums +it() bf(rsync_delta.) data blocks for file update & change +) + +See bf(http://www.ils.unc.edu/i2dsi/unc_rsync+.html) for papers and technical +reports. + manpagesection(DIAGNOSTICS) rsync occasionally produces error messages that may seem a little diff --git a/sender.c b/sender.c index d2ab1b34..39bb2797 100644 --- a/sender.c +++ b/sender.c @@ -93,6 +93,14 @@ void send_files(struct file_list *flist,int f_out,int f_in) int phase = 0; extern struct stats stats; struct stats initial_stats; + extern int write_batch; /* dw */ + int negative_one; /* dw */ + extern int read_batch; /* dw */ + int checksums_match; /* dw */ + int buff_len; /* dw */ + char buff[CHUNK_SIZE]; /* dw */ + int j; /* dw */ + int done; /* dw */ if (verbose > 2) rprintf(FINFO,"send_files starting\n"); @@ -152,12 +160,15 @@ void send_files(struct file_list *flist,int f_out,int f_in) initial_stats = stats; s = receive_sums(f_in); + if (write_batch) /* dw */ + write_batch_csum_info(&i,flist->count,s); if (!s) { io_error = 1; rprintf(FERROR,"receive_sums failed\n"); return; } + if (!read_batch) { fd = do_open(fname, O_RDONLY, 0); if (fd == -1) { io_error = 1; @@ -185,28 +196,78 @@ void send_files(struct file_list *flist,int f_out,int f_in) if (verbose > 2) rprintf(FINFO,"send_files mapped %s of size %.0f\n", fname,(double)st.st_size); + } - write_int(f_out,i); + if (!read_batch) { /* dw */ + write_int(f_out,i); - write_int(f_out,s->count); - write_int(f_out,s->n); - write_int(f_out,s->remainder); + if (write_batch) + write_batch_delta_file((char *)&i,sizeof(i)); + + write_int(f_out,s->count); + write_int(f_out,s->n); + write_int(f_out,s->remainder); + } if (verbose > 2) - rprintf(FINFO,"calling match_sums %s\n",fname); + if (!read_batch) + rprintf(FINFO,"calling match_sums %s\n",fname); if (!am_server) { log_transfer(file, fname+offset); } set_compression(fname); - - match_sums(f_out,s,buf,st.st_size); - log_send(file, &initial_stats); - - if (buf) unmap_file(buf); - close(fd); + if (read_batch) { /* dw */ + /* read checksums originally computed on sender side */ + read_batch_csum_info(i, s, &checksums_match); + if (checksums_match) { + read_batch_delta_file( (char *) &j, sizeof(int) ); + if (j != i) { /* if flist index entries don't match*/ + rprintf(FINFO,"index mismatch in send_files\n"); + rprintf(FINFO,"read index = %d flist ndx = %d\n",j,i); + close_batch_delta_file(); + close_batch_csums_file(); + exit_cleanup(1); + } + else { + write_int(f_out,j); + write_int(f_out,s->count); + write_int(f_out,s->n); + write_int(f_out,s->remainder); + done=0; + while (!done) { + read_batch_delta_file( (char *) &buff_len, sizeof(int) ); + write_int(f_out,buff_len); + if (buff_len == 0) { + done = 1; + } + else { + if (buff_len > 0) { + read_batch_delta_file(buff, buff_len); + write_buf(f_out,buff,buff_len); + } + } + } /* end while */ + read_batch_delta_file( buff, MD4_SUM_LENGTH); + write_buf(f_out, buff, MD4_SUM_LENGTH); + + } /* j=i */ + } else { /* not checksum match */ + rprintf(FINFO,"readbatch & checksums don't match\n"); + rprintf(FINFO,"filename=%s is being skipped\n"); + continue; + } + } else { + match_sums(f_out,s,buf,st.st_size); + log_send(file, &initial_stats); + } + + if (!read_batch) { /* dw */ + if (buf) unmap_file(buf); + close(fd); + } free_sums(s); @@ -220,6 +281,11 @@ void send_files(struct file_list *flist,int f_out,int f_in) match_report(); write_int(f_out,-1); + if (write_batch || read_batch) { /* dw */ + close_batch_csums_file(); + close_batch_delta_file(); + } + } diff --git a/token.c b/token.c index 2967b44c..174f121a 100644 --- a/token.c +++ b/token.c @@ -90,18 +90,29 @@ static int simple_recv_token(int f,char **data) static void simple_send_token(int f,int token, struct map_struct *buf,OFF_T offset,int n) { + extern int write_batch; /* dw */ + int hold_int; /* dw */ + if (n > 0) { int l = 0; while (l < n) { int n1 = MIN(CHUNK_SIZE,n-l); write_int(f,n1); write_buf(f,map_ptr(buf,offset+l,n1),n1); + if (write_batch) { + write_batch_delta_file( (char *) &n1, sizeof(int) ); + write_batch_delta_file(map_ptr(buf,offset+l,n1),n1); + } l += n1; } } /* a -2 token means to send data only and no token */ if (token != -2) { write_int(f,-(token+1)); + if (write_batch) { + hold_int = -(token+1); + write_batch_delta_file( (char *) &hold_int, sizeof(int) ); + } } } @@ -134,6 +145,8 @@ send_deflated_token(int f, int token, { int n, r; static int init_done, flush_pending; + extern int write_batch; /* dw */ + char temp_byte; /* dw */ if (last_token == -1) { /* initialization */ @@ -166,13 +179,27 @@ send_deflated_token(int f, int token, n = last_token - run_start; if (r >= 0 && r <= 63) { write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r); + if (write_batch) { /* dw */ + temp_byte = (char)( (n==0? TOKEN_REL: TOKENRUN_REL) + r); + write_batch_delta_file(&temp_byte,sizeof(char)); + } } else { write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG)); write_int(f, run_start); + if (write_batch) { /* dw */ + temp_byte = (char)(n==0? TOKEN_LONG: TOKENRUN_LONG); + write_batch_delta_file(&temp_byte,sizeof(temp_byte)); + write_batch_delta_file((char *)&run_start,sizeof(run_start)); + } } if (n != 0) { write_byte(f, n); write_byte(f, n >> 8); + if (write_batch) { /* dw */ + write_batch_delta_file((char *)&n,sizeof(char)); + temp_byte = (char) n >> 8; + write_batch_delta_file(&temp_byte,sizeof(temp_byte)); + } } last_run_end = last_token; run_start = token; @@ -231,6 +258,8 @@ send_deflated_token(int f, int token, obuf[0] = DEFLATED_DATA + (n >> 8); obuf[1] = n; write_buf(f, obuf, n+2); + if (write_batch) /* dw */ + write_batch_delta_file(obuf,n+2); } } } while (nb != 0 || tx_strm.avail_out == 0); @@ -240,6 +269,10 @@ send_deflated_token(int f, int token, if (token == -1) { /* end of file - clean up */ write_byte(f, END_FLAG); + if (write_batch) { /* dw */ + temp_byte = END_FLAG; + write_batch_delta_file((char *)&temp_byte,sizeof(temp_byte)); + } } else if (token != -2) { /* add the data in the current block to the compressor's diff --git a/util.c b/util.c index 50fdaf8e..ada40465 100644 --- a/util.c +++ b/util.c @@ -154,6 +154,7 @@ pid_t local_child(int argc, char **argv,int *f_in,int *f_out) pid_t pid; int to_child_pipe[2]; int from_child_pipe[2]; + extern int read_batch; /* dw */ if (fd_pair(to_child_pipe) < 0 || fd_pair(from_child_pipe) < 0) { @@ -172,7 +173,10 @@ pid_t local_child(int argc, char **argv,int *f_in,int *f_out) extern int am_sender; extern int am_server; - am_sender = !am_sender; + if (read_batch) + am_sender = 0; + else + am_sender = !am_sender; am_server = 1; if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 || -- 2.34.1