From 088aac85971f3f1571c7f90569c95d5025b1fd82 Mon Sep 17 00:00:00 2001 From: David Dykstra Date: Wed, 6 Feb 2002 21:20:48 +0000 Subject: [PATCH] Make batch mode actually work and add man page documentation. From Jos Backus. --- NEWS | 4 +- batch.c | 232 +++++++++++++++++++++++++++--------------------------- errcode.h | 1 + log.c | 17 ++-- main.c | 63 ++++++++------- options.c | 57 +++++++++----- rsync.yo | 98 ++++++++++++++++++----- 7 files changed, 275 insertions(+), 197 deletions(-) diff --git a/NEWS b/NEWS index 620db73a..8c54134c 100644 --- a/NEWS +++ b/NEWS @@ -7,5 +7,7 @@ rsync 2.5.3 (not released yet) what was historically used so as not to break scripts that try to parse the output. - * Added --no-whole-file and --no-blocking-io options + * Added --no-whole-file and --no-blocking-io options (Dave Dykstra) + * Made the --write-batch and --read-batch options actually work + and added documentation in the man page (Jos Backus) diff --git a/batch.c b/batch.c index a470710d..237d57bd 100644 --- a/batch.c +++ b/batch.c @@ -8,55 +8,38 @@ #include "rsync.h" #include -char rsync_flist_file[27] = "rsync_flist."; -char rsync_csums_file[27] = "rsync_csums."; -char rsync_delta_file[27] = "rsync_delta."; -char rsync_argvs_file[27] = "rsync_argvs."; - -char batch_file_ext[15]; - -int fdb; -int fdb_delta; -int fdb_open; -int fdb_close; +extern char *batch_prefix; struct file_list *batch_flist; -void create_batch_file_ext() -{ - struct tm *timeptr; - time_t elapsed_seconds; - - /* Save run date and time to use for batch file extensions */ - time(&elapsed_seconds); - timeptr = localtime(&elapsed_seconds); - - sprintf(batch_file_ext, "%4d%02d%02d%02d%02d%02d", - timeptr->tm_year + 1900, timeptr->tm_mon + 1, - timeptr->tm_mday, timeptr->tm_hour, timeptr->tm_min, - timeptr->tm_sec); - rprintf(FINFO,"batch file extension: %s\n", batch_file_ext); -} +static char rsync_flist_file[] = ".rsync_flist"; +static char rsync_csums_file[] = ".rsync_csums"; +static char rsync_delta_file[] = ".rsync_delta"; +static char rsync_argvs_file[] = ".rsync_argvs"; -void set_batch_file_ext(char *ext) -{ - strcpy(batch_file_ext, ext); -} +static int fdb; +static int fdb_delta; +static int fdb_open; +static int fdb_close; void write_batch_flist_file(char *buff, int bytes_to_write) { + char filename[MAXPATHLEN]; if (fdb_open) { /* Set up file extension */ - strcat(rsync_flist_file, batch_file_ext); + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_flist_file, sizeof(filename)); - /* Open batch flist file for writing; create it if it doesn't exist */ - fdb = - do_open(rsync_flist_file, O_WRONLY | O_CREAT | O_TRUNC, + /* + * Open batch flist file for writing; + * create it if it doesn't exist + */ + fdb = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IREAD | S_IWRITE); if (fdb == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_flist_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } @@ -67,12 +50,11 @@ void write_batch_flist_file(char *buff, int bytes_to_write) if (write(fdb, buff, bytes_to_write) == -1) { rprintf(FERROR, "Batch file %s write error: %s\n", - rsync_flist_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } - if (fdb_close) { close(fdb); } @@ -110,7 +92,6 @@ void write_batch_flist_info(int flist_count, struct file_struct **fptr) } write_char_bufs(fptr[i]->sum); } - } void write_char_bufs(char *buf) @@ -118,11 +99,8 @@ void write_char_bufs(char *buf) /* Write the size of the string which will follow */ char b[4]; - if (buf != NULL) - SIVAL(b, 0, strlen(buf)); - else { - SIVAL(b, 0, 0); - } + + SIVAL(b, 0, buf != NULL ? strlen(buf) : 0); write_batch_flist_file(b, sizeof(int)); @@ -137,36 +115,52 @@ void write_batch_argvs_file(int argc, char *argv[]) { int fdb; int i; - char buff[256]; - - strcat(rsync_argvs_file, batch_file_ext); - - - /* Open batch argvs file for writing; create it if it doesn't exist */ - fdb = do_open(rsync_argvs_file, O_WRONLY | O_CREAT | O_TRUNC, + char buff[256]; /* XXX */ + char buff2[MAXPATHLEN + 6]; + char filename[MAXPATHLEN]; + + /* Set up file extension */ + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_argvs_file, sizeof(filename)); + + /* + * Open batch argvs file for writing; + * create it if it doesn't exist + */ + fdb = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IREAD | S_IWRITE | S_IEXEC); if (fdb == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_argvs_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } buff[0] = '\0'; + /* Write argvs info to batch file */ for (i = 0; i < argc; ++i) { - if (i == argc - 2) + if (i == argc - 2) /* Skip source directory on cmdline */ continue; /* * FIXME: * I think directly manipulating argv[] is probably bogus */ - if (!strcmp(argv[i], "--write-batch")) { + if (!strncmp(argv[i], "--write-batch", + strlen("--write-batch"))) { /* Safer to change it here than script */ - /* Change to --read-batch + ext * to get ready for remote */ - strlcat(buff, "--read-batch ", sizeof(buff)); - strlcat(buff, batch_file_ext, sizeof(buff)); - } else { + /* + * Change to --read-batch=prefix + * to get ready for remote + */ + strlcat(buff, "--read-batch=", sizeof(buff)); + strlcat(buff, batch_prefix, sizeof(buff)); + } else + if (i == argc - 1) { + snprintf(buff2, sizeof(buff2), "${1:-%s}", argv[i]); + strlcat(buff, buff2, sizeof(buff)); + } + else { strlcat(buff, argv[i], sizeof(buff)); } @@ -177,14 +171,14 @@ void write_batch_argvs_file(int argc, char *argv[]) strlcat(buff, "\n", sizeof(buff)); if (!write(fdb, buff, strlen(buff))) { rprintf(FERROR, "Batch file %s write error: %s\n", - rsync_argvs_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } close(fdb); } -struct file_list *create_flist_from_batch() +struct file_list *create_flist_from_batch(void) { unsigned char flags; @@ -201,7 +195,7 @@ struct file_list *create_flist_from_batch() (struct file_struct **) malloc(sizeof(batch_flist->files[0]) * batch_flist->malloced); if (!batch_flist->files) { - out_of_memory("create_flist_from_batch"); /* dw -- will exit */ + out_of_memory("create_flist_from_batch"); } for (flags = read_batch_flags(); flags; flags = read_batch_flags()) { @@ -231,23 +225,23 @@ struct file_list *create_flist_from_batch() } return batch_flist; - } int read_batch_flist_file(char *buff, int len) { int bytes_read; + char filename[MAXPATHLEN]; if (fdb_open) { - - /* Set up file extension */ - strcat(rsync_flist_file, batch_file_ext); + /* Set up file extension */ + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_flist_file, sizeof(filename)); /* Open batch flist file for reading */ - fdb = do_open(rsync_flist_file, O_RDONLY, 0); + fdb = do_open(filename, O_RDONLY, 0); if (fdb == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_flist_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } @@ -256,17 +250,17 @@ int read_batch_flist_file(char *buff, int len) /* Read flist batch file */ - bytes_read = read(fdb, buff, len); - - if (bytes_read == -1) { + switch (bytes_read = read(fdb, buff, len)) { + case -1: rprintf(FERROR, "Batch file %s read error: %s\n", - rsync_flist_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); - } - if (bytes_read == 0) { /* EOF */ + break; + case 0: /* EOF */ close(fdb); } + return bytes_read; } @@ -293,8 +287,12 @@ void read_batch_flist_info(struct file_struct **fptr) out_of_memory("read_batch_flist_info"); memset((char *) file, 0, sizeof(*file)); - (*fptr) = file; + *fptr = file; + /* + * Keep these in sync with bytes_to_write assignment + * in write_batch_flist_info() + */ read_batch_flist_file((char *) &file->modtime, sizeof(time_t)); read_batch_flist_file((char *) &file->length, sizeof(OFF_T)); read_batch_flist_file((char *) &file->mode, sizeof(mode_t)); @@ -356,20 +354,23 @@ void read_batch_flist_info(struct file_struct **fptr) void write_batch_csums_file(void *buff, int bytes_to_write) { - static int fdb_open = 1; + char filename[MAXPATHLEN]; if (fdb_open) { /* Set up file extension */ - strcat(rsync_csums_file, batch_file_ext); + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_csums_file, sizeof(filename)); - /* Open batch csums file for writing; create it if it doesn't exist */ - fdb = - do_open(rsync_csums_file, O_WRONLY | O_CREAT | O_TRUNC, + /* + * Open batch csums file for writing; + * create it if it doesn't exist + */ + fdb = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IREAD | S_IWRITE); if (fdb == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_csums_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } @@ -380,16 +381,15 @@ void write_batch_csums_file(void *buff, int bytes_to_write) if (write(fdb, buff, bytes_to_write) == -1) { rprintf(FERROR, "Batch file %s write error: %s\n", - rsync_csums_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } } -void close_batch_csums_file() +void close_batch_csums_file(void) { close(fdb); - } void write_batch_csum_info(int *flist_entry, int flist_count, @@ -405,10 +405,7 @@ void write_batch_csum_info(int *flist_entry, int flist_count, /* FIXME: This will break if s->count is ever not exactly an int. */ write_batch_csums_file(flist_entry, sizeof(int)); - if (s) - write_batch_csums_file(&s->count, sizeof(int)); - else - write_batch_csums_file(&int_zero, sizeof (int)); + write_batch_csums_file(s ? &s->count : &int_zero, sizeof(int)); if (s) { for (i = 0; i < s->count; i++) { @@ -417,8 +414,7 @@ void write_batch_csum_info(int *flist_entry, int flist_count, && (i == s->count - 1)) { fdb_close = 1; } - write_batch_csums_file(s->sums[i].sum2, - csum_length); + write_batch_csums_file(s->sums[i].sum2, csum_length); } } } @@ -427,17 +423,18 @@ int read_batch_csums_file(char *buff, int len) { static int fdb_open = 1; int bytes_read; + char filename[MAXPATHLEN]; if (fdb_open) { - - /* Set up file extension */ - strcat(rsync_csums_file, batch_file_ext); + /* Set up file extension */ + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_csums_file, sizeof(filename)); /* Open batch flist file for reading */ - fdb = do_open(rsync_csums_file, O_RDONLY, 0); + fdb = do_open(filename, O_RDONLY, 0); if (fdb == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_csums_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } @@ -450,14 +447,14 @@ int read_batch_csums_file(char *buff, int len) if (bytes_read == -1) { rprintf(FERROR, "Batch file %s read error: %s\n", - rsync_csums_file, strerror(errno)); + filename, strerror(errno)); close(fdb); exit_cleanup(1); } + return bytes_read; } - void read_batch_csum_info(int flist_entry, struct sum_struct *s, int *checksums_match) { @@ -468,11 +465,9 @@ void read_batch_csum_info(int flist_entry, struct sum_struct *s, char file_sum2[SUM_LENGTH]; extern int csum_length; - read_batch_csums_file((char *) &file_flist_entry, sizeof(int)); if (file_flist_entry != flist_entry) { - rprintf(FINFO, "file_list_entry NE flist_entry\n"); - rprintf(FINFO, "file_flist_entry = %d flist_entry = %d\n", + rprintf(FINFO, "file_flist_entry (%d) != flist_entry (%d)\n", file_flist_entry, flist_entry); close(fdb); exit_cleanup(1); @@ -488,31 +483,33 @@ void read_batch_csum_info(int flist_entry, struct sum_struct *s, read_batch_csums_file(file_sum2, csum_length); if ((s->sums[i].sum1 != file_sum1) || - (memcmp - (s->sums[i].sum2, file_sum2, - csum_length) != 0)) { + (memcmp(s->sums[i].sum2, file_sum2, csum_length) + != 0)) { *checksums_match = 0; } } /* end for */ } - } void write_batch_delta_file(char *buff, int bytes_to_write) { static int fdb_delta_open = 1; + char filename[MAXPATHLEN]; if (fdb_delta_open) { /* Set up file extension */ - strcat(rsync_delta_file, batch_file_ext); + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_delta_file, sizeof(filename)); - /* Open batch delta file for writing; create it if it doesn't exist */ - fdb_delta = - do_open(rsync_delta_file, O_WRONLY | O_CREAT | O_TRUNC, + /* + * Open batch delta file for writing; + * create it if it doesn't exist + */ + fdb_delta = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IREAD | S_IWRITE); if (fdb_delta == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_delta_file, strerror(errno)); + filename, strerror(errno)); close(fdb_delta); exit_cleanup(1); } @@ -523,32 +520,33 @@ void write_batch_delta_file(char *buff, int bytes_to_write) if (write(fdb_delta, buff, bytes_to_write) == -1) { rprintf(FERROR, "Batch file %s write error: %s\n", - rsync_delta_file, strerror(errno)); + filename, strerror(errno)); close(fdb_delta); exit_cleanup(1); } } -void close_batch_delta_file() + +void close_batch_delta_file(void) { close(fdb_delta); - } int read_batch_delta_file(char *buff, int len) { static int fdb_delta_open = 1; int bytes_read; + char filename[MAXPATHLEN]; if (fdb_delta_open) { - - /* Set up file extension */ - strcat(rsync_delta_file, batch_file_ext); + /* Set up file extension */ + strlcpy(filename, batch_prefix, sizeof(filename)); + strlcat(filename, rsync_delta_file, sizeof(filename)); /* Open batch flist file for reading */ - fdb_delta = do_open(rsync_delta_file, O_RDONLY, 0); + fdb_delta = do_open(filename, O_RDONLY, 0); if (fdb_delta == -1) { rprintf(FERROR, "Batch file %s open error: %s\n", - rsync_delta_file, strerror(errno)); + filename, strerror(errno)); close(fdb_delta); exit_cleanup(1); } @@ -561,14 +559,14 @@ int read_batch_delta_file(char *buff, int len) if (bytes_read == -1) { rprintf(FERROR, "Batch file %s read error: %s\n", - rsync_delta_file, strerror(errno)); + filename, strerror(errno)); close(fdb_delta); exit_cleanup(1); } + return bytes_read; } - void show_flist(int index, struct file_struct **fptr) { /* for debugging show_flist(flist->count, flist->files * */ diff --git a/errcode.h b/errcode.h index fc1eca78..18d04143 100644 --- a/errcode.h +++ b/errcode.h @@ -26,6 +26,7 @@ #define RERR_PROTOCOL 2 /* protocol incompatibility */ #define RERR_FILESELECT 3 /* errors selecting input/output files, dirs */ #define RERR_UNSUPPORTED 4 /* requested action not supported */ +#define RERR_STARTCLIENT 5 /* error starting client-server protocol */ #define RERR_SOCKETIO 10 /* error in socket IO */ #define RERR_FILEIO 11 /* error in file IO */ diff --git a/log.c b/log.c index 045f0317..f07b64b1 100644 --- a/log.c +++ b/log.c @@ -41,6 +41,7 @@ struct { { RERR_PROTOCOL , "protocol incompatibility" }, { RERR_FILESELECT , "errors selecting input/output files, dirs" }, { RERR_UNSUPPORTED, "requested action not supported" }, + { RERR_STARTCLIENT, "error starting client-server protocol" }, { RERR_SOCKETIO , "error in socket IO" }, { RERR_FILEIO , "error in file IO" }, { RERR_STREAMIO , "error in rsync protocol data stream" }, @@ -559,20 +560,16 @@ void log_exit(int code, const char *file, int line) } } - - - -/* log the incoming transfer of a file for interactive use, this - will be called at the end where the client was run - - it i called when a file starts to be transferred -*/ +/* + * Log the incoming transfer of a file for interactive use, + * this will be called at the end where the client was run. + * Called when a file starts to be transferred. + */ void log_transfer(struct file_struct *file, const char *fname) { extern int verbose; if (!verbose) return; - rprintf(FINFO,"%s\n", fname); + rprintf(FINFO, "%s\n", fname); } - diff --git a/main.c b/main.c index fa5ecc03..0ed7df38 100644 --- a/main.c +++ b/main.c @@ -177,7 +177,7 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int extern int blocking_io; extern int read_batch; - if (!read_batch && !local_server) { /* dw -- added read_batch */ + if (!read_batch && !local_server) { if (!cmd) cmd = getenv(RSYNC_RSH_ENV); if (!cmd) @@ -230,7 +230,7 @@ static pid_t do_cmd(char *cmd,char *machine,char *user,char *path,int *f_in,int if (local_server) { if (read_batch) - create_flist_from_batch(); + create_flist_from_batch(); /* sets batch_flist */ ret = local_child(argc, args, f_in, f_out); } else { ret = piped_child(args,f_in,f_out); @@ -443,8 +443,8 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) extern int am_daemon; extern int module_id; extern int am_sender; - extern int read_batch; /* dw */ - extern struct file_list *batch_flist; /* dw */ + extern int read_batch; + extern struct file_list *batch_flist; if (verbose > 2) rprintf(FINFO,"server_recv(%d) starting pid=%d\n",argc,(int)getpid()); @@ -470,7 +470,7 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) if (delete_mode && !delete_excluded) recv_exclude_list(f_in); - if (read_batch) /* dw */ + if (read_batch) flist = batch_flist; else flist = recv_file_list(f_in); @@ -497,7 +497,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[]) extern int cvs_exclude; extern int am_sender; extern int remote_version; - extern int read_batch; /* dw */ + extern int read_batch; setup_protocol(f_out, f_in); @@ -508,7 +508,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[]) io_start_multiplex_out(f_out); if (am_sender) { - if (!read_batch) { /* dw */ + if (!read_batch) { recv_exclude_list(f_in); if (cvs_exclude) add_cvs_excludes(); @@ -527,19 +527,19 @@ void start_server(int f_in, int f_out, int argc, char *argv[]) */ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) { - struct file_list *flist; + struct file_list *flist = NULL; int status = 0, status2 = 0; char *local_name = NULL; extern int am_sender; extern int remote_version; extern pid_t cleanup_child_pid; - extern int write_batch; /* dw */ - extern int read_batch; /* dw */ - extern struct file_list *batch_flist; /* dw */ + extern int write_batch; + extern int read_batch; + extern struct file_list *batch_flist; cleanup_child_pid = pid; if (read_batch) - flist = batch_flist; /* dw */ + flist = batch_flist; set_nonblocking(f_in); set_nonblocking(f_out); @@ -582,7 +582,7 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) list_only = 1; } - if (!write_batch) /* dw */ + if (!write_batch) send_exclude_list(f_out); flist = recv_file_list(f_in); @@ -658,6 +658,7 @@ static int start_client(int argc, char *argv[]) extern char *shell_cmd; extern int rsync_port; extern int whole_file; + extern int write_batch; extern int read_batch; int rc; @@ -685,7 +686,7 @@ static int start_client(int argc, char *argv[]) return start_socket_client(host, path, argc-1, argv+1); } - if (!read_batch) { /* dw */ + if (!read_batch) { p = find_colon(argv[0]); if (p) { @@ -711,8 +712,11 @@ static int start_client(int argc, char *argv[]) p = find_colon(argv[argc-1]); if (!p) { local_server = 1; - /* disable "rsync algorithm" when both sides local */ - if (whole_file == -1) + /* + * disable "rsync algorithm" when both sides local, + * except when creating a batch update + */ + if (!write_batch && whole_file == -1) whole_file = 1; } else if (p[1] == ':') { *p = 0; @@ -735,9 +739,9 @@ static int start_client(int argc, char *argv[]) argc--; } } else { - am_sender = 1; /* dw */ - local_server = 1; /* dw */ - shell_path = argv[argc-1]; /* dw */ + am_sender = 1; + local_server = 1; + shell_path = argv[argc-1]; } if (shell_machine) { @@ -802,13 +806,11 @@ int main(int argc,char *argv[]) extern int am_daemon; extern int am_server; int ret; - extern int read_batch; /* dw */ - extern int write_batch; /* dw */ - extern char *batch_ext; /* dw */ - int orig_argc; /* dw */ + extern int write_batch; + int orig_argc; char **orig_argv; - orig_argc = argc; /* dw */ + orig_argc = argc; orig_argv = argv; signal(SIGUSR1, sigusr1_handler); @@ -847,15 +849,10 @@ int main(int argc,char *argv[]) that implement getcwd that way "pwd" can't be found after chroot. */ push_dir(NULL,0); - if (write_batch) { /* dw */ - create_batch_file_ext(); + if (write_batch && !am_server) { write_batch_argvs_file(orig_argc, orig_argv); } - if (read_batch) { /* dw */ - set_batch_file_ext(batch_ext); - } - if (am_daemon) { return daemon_main(); } @@ -882,7 +879,9 @@ int main(int argc,char *argv[]) } ret = start_client(argc, argv); - exit_cleanup(ret); + if (ret == -1) + exit_cleanup(RERR_STARTCLIENT); + else + exit_cleanup(ret); return ret; } - diff --git a/options.c b/options.c index 98673cc1..6a73e3c6 100644 --- a/options.c +++ b/options.c @@ -87,9 +87,8 @@ int default_af_hint = AF_INET; /* Must use IPv4 */ * or under Unix process-monitors. **/ int no_detach = 0; - -int read_batch=0; -int write_batch=0; +int write_batch = 0; +int read_batch = 0; char *backup_suffix = BACKUP_SUFFIX; char *tmpdir = NULL; @@ -107,7 +106,7 @@ int quiet = 0; int always_checksum = 0; int list_only = 0; -char *batch_ext = NULL; +char *batch_prefix = NULL; static int modify_window_set; @@ -244,8 +243,8 @@ void usage(enum logcode F) rprintf(F," --log-format=FORMAT log file transfers using specified format\n"); rprintf(F," --password-file=FILE get password from FILE\n"); rprintf(F," --bwlimit=KBPS limit I/O bandwidth, KBytes per second\n"); - rprintf(F," --read-batch=EXT read batch file\n"); - rprintf(F," --write-batch write batch file\n"); + rprintf(F," --write-batch=PREFIX write batch fileset starting with PREFIX\n"); + rprintf(F," --read-batch=PREFIX read batch fileset starting with PREFIX\n"); rprintf(F," -h, --help show this help screen\n"); #ifdef INET6 rprintf(F," -4 prefer IPv4\n"); @@ -339,8 +338,8 @@ static struct poptOption long_options[] = { {"address", 0, POPT_ARG_STRING, &bind_address, 0}, {"backup-dir", 0, POPT_ARG_STRING, &backup_dir}, {"hard-links", 'H', POPT_ARG_NONE, &preserve_hard_links}, - {"read-batch", 0, POPT_ARG_STRING, &batch_ext, OPT_READ_BATCH}, - {"write-batch", 0, POPT_ARG_NONE, &write_batch}, + {"read-batch", 0, POPT_ARG_STRING, &batch_prefix, OPT_READ_BATCH}, + {"write-batch", 0, POPT_ARG_STRING, &batch_prefix, OPT_WRITE_BATCH}, #ifdef INET6 {0, '4', POPT_ARG_VAL, &default_af_hint, AF_INET }, {0, '6', POPT_ARG_VAL, &default_af_hint, AF_INET6 }, @@ -523,8 +522,13 @@ int parse_arguments(int *argc, const char ***argv, int frommain) keep_partial = 1; break; + case OPT_WRITE_BATCH: + /* popt stores the filename in batch_prefix for us */ + write_batch = 1; + break; + case OPT_READ_BATCH: - /* The filename is stored in batch_ext for us by popt */ + /* popt stores the filename in batch_prefix for us */ read_batch = 1; break; @@ -540,6 +544,22 @@ int parse_arguments(int *argc, const char ***argv, int frommain) } } + if (write_batch && read_batch) { + snprintf(err_buf,sizeof(err_buf), + "write-batch and read-batch can not be used together\n"); + rprintf(FERROR,"ERROR: write-batch and read-batch" + " can not be used together\n"); + return 0; + } + + if (do_compression && (write_batch || read_batch)) { + snprintf(err_buf,sizeof(err_buf), + "compress can not be used with write-batch or read-batch\n"); + rprintf(FERROR,"ERROR: compress can not be used with" + " write-batch or read-batch\n"); + return 0; + } + *argv = poptGetArgs(pc); if (*argv) *argc = count_args(*argv); @@ -561,8 +581,8 @@ void server_options(char **args,int *argc) static char mdelete[30]; static char mwindow[30]; static char bw[50]; - static char fext[20]; - static char wbatch[14]; + /* Leave room for ``--(write|read)-batch='' */ + static char fext[MAXPATHLEN + 15]; int i, x; @@ -644,13 +664,14 @@ void server_options(char **args,int *argc) args[ac++] = mdelete; } - if (write_batch) { - snprintf(wbatch,sizeof(wbatch),"--write-batch"); - args[ac++] = wbatch; - } - - if (batch_ext != NULL) { - snprintf(fext,sizeof(fext),"--read-batch=%s",batch_ext); + if (batch_prefix != NULL) { + char *fmt = ""; + if (write_batch) + fmt = "--write-batch=%s"; + else + if (read_batch) + fmt = "--read-batch=%s"; + snprintf(fext,sizeof(fext),fmt,batch_prefix); args[ac++] = fext; } diff --git a/rsync.yo b/rsync.yo index 035a352c..6282e635 100644 --- a/rsync.yo +++ b/rsync.yo @@ -280,8 +280,8 @@ verb( --log-format=FORMAT log file transfers using specified format --password-file=FILE get password from FILE --bwlimit=KBPS limit I/O bandwidth, KBytes per second - --read-batch=FILE read batch file - --write-batch write batch file + --read-batch=PREFIX read batch fileset starting with PREFIX + --write-batch=PREFIX write batch fileset starting with PREFIX -h, --help show this help screen @@ -706,10 +706,13 @@ transfer was too fast, it will wait before sending the next data block. The result is an average transfer rate equalling the specified limit. A value of zero specifies no limit. -dit(bf(--read-batch)) Apply a previously generated change batch. +dit(bf(--write-batch=PREFIX)) Generate a set of files that can be +transferred as a batch update. Each filename in the set starts with +PREFIX. See the "BATCH MODE" section for details. -dit(bf(--write-batch)) Generate a set of files that can be transferred -as a batch update. +dit(bf(--read-batch=PREFIX)) Apply a previously generated change batch, +using the fileset whose filenames start with PREFIX. See the "BATCH +MODE" section for details. enddit() @@ -804,26 +807,83 @@ itemize( manpagesection(BATCH MODE) bf(Note:) Batch mode should be considered experimental in this version -of rsync. The interface or behaviour may change before it stabilizes. +of rsync. The interface or behaviour may change before it stabilizes. + +Batch mode can be used to apply the same set of updates to many +identical systems. Suppose one has a tree which is replicated on a +number of hosts. Now suppose some changes have been made to this +source tree and those changes need to be propagated to the other +hosts. In order to do this using batch mode, rsync is run with the +write-batch option to apply the changes made to the source tree to one +of the destination trees. The write-batch option causes the rsync +client to store the information needed to repeat this operation against +other destination trees in a batch update fileset (see below). The +filename of each file in the fileset starts with a prefix specified by +the user as an argument to the write-batch option. This fileset is +then copied to each remote host, where rsync is run with the read-batch +option, again specifying the same prefix, and the destination tree. +Rsync updates the destination tree using the information stored in the +batch update fileset. + +The fileset consists of 4 files: -The following call generates 4 files that encapsulate the information -for synchronizing the contents of bf(target_dir) with the updates found in -bf(src_dir) - -quote( -$ rsync --write-batch [other rsync options here] \nl() - /somewhere/src_dir /somewhere/target_dir +itemize( +it() bf(.rsync_argvs) command-line arguments +it() bf(.rsync_flist) rsync internal file metadata +it() bf(.rsync_csums) rsync checksums +it() bf(.rsync_delta) data blocks for file update & change ) -The generated files are labeled with a common timestamp: +The .rsync_argvs file contains a command-line suitable for updating a +destination tree using that batch update fileset. It can be executed +using a Bourne(-like) shell, optionally passing in an alternate +destination tree pathname which is then used instead of the original +path. This is useful when the destination tree path differs from the +original destination tree path. -itemize( -it() bf(rsync_argvs.) command-line arguments -it() bf(rsync_flist.) rsync internal file metadata -it() bf(rsync_csums.) rsync checksums -it() bf(rsync_delta.) data blocks for file update & change +Generating the batch update fileset once saves having to perform the +file status, checksum and data block generation more than once when +updating multiple destination trees. Multicast transport protocols can +be used to transfer the batch update files in parallel to many hosts at +once, instead of sending the same data to every host individually. + +Example: + +verb( +$ rsync --write_batch=pfx -a /source/dir/ /adest/dir/ +$ rcp pfx.rsync_* remote: +$ rsh remote rsync --read_batch=pfx -a /bdest/dir/ +# or alternatively +$ rsh remote ./pfx.rsync_argvs /bdest/dir/ ) +In this example, rsync is used to update /adest/dir/ with /source/dir/ +and the information to repeat this operation is stored in the files +pfx.rsync_*. These files are then copied to the machine named "remote". +Rsync is then invoked on "remote" to update /bdest/dir/ the same way as +/adest/dir/. The last line shows the rsync_argvs file being used to +invoke rsync. + +Caveats: + +The read-batch option expects the destination tree it is meant to update +to be identical to the destination tree that was used to create the +batch update fileset. When a difference between the destination trees +is encountered the update will fail at that point, leaving the +destination tree in a partially updated state. In that case, rsync can +be used in its regular (non-batch) mode of operation to fix up the +destination tree. + +The rsync version used on all destinations should be identical to the +one used on the original destination. + +The -z/--compress option does not work in batch mode and yields a usage +error. A separate compression tool can be used instead to reduce the +size of the batch update files for transport to the destination. + +The -n/--dryrun option does not work in batch mode and yields a runtime +error. + See bf(http://www.ils.unc.edu/i2dsi/unc_rsync+.html) for papers and technical reports. -- 2.34.1