A new option that allows the creation of a batched transfer
authorWayne Davison <wayned@samba.org>
Sun, 27 Mar 2005 05:16:19 +0000 (05:16 +0000)
committerWayne Davison <wayned@samba.org>
Sun, 27 Mar 2005 05:16:19 +0000 (05:16 +0000)
without changing the destination.  Particularly useful with
a push.

only-write-batch.diff [new file with mode: 0644]

diff --git a/only-write-batch.diff b/only-write-batch.diff
new file mode 100644 (file)
index 0000000..1bc43bb
--- /dev/null
@@ -0,0 +1,366 @@
+--- orig/batch.c       2005-03-27 05:13:14
++++ batch.c    2005-03-27 05:14:42
+@@ -139,7 +139,7 @@ static void write_filter_rules(int fd)
+  * (hopefully) work. */
+ void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt)
+ {
+-      int fd, i;
++      int fd, i, len;
+       char *p, filename[MAXPATHLEN];
+       stringjoin(filename, sizeof filename,
+@@ -175,11 +175,12 @@ void write_batch_shell_file(int argc, ch
+                       continue;
+               }
+               write(fd, " ", 1);
+-              if (strncmp(p, "--write-batch", 13) == 0) {
++              if (strncmp(p, "--write-batch", len = 13) == 0
++               || strncmp(p, "--only-write-batch", len = 18) == 0) {
+                       write(fd, "--read-batch", 12);
+-                      if (p[13] == '=') {
++                      if (p[len] == '=') {
+                               write(fd, "=", 1);
+-                              write_arg(fd, p + 14);
++                              write_arg(fd, p + len + 1);
+                       }
+               } else
+                       write_arg(fd, p);
+--- orig/generator.c   2005-03-17 09:05:21
++++ generator.c        2005-03-24 07:07:12
+@@ -25,6 +25,7 @@
+ extern int verbose;
+ extern int dry_run;
++extern int do_xfers;
+ extern int log_format_has_i;
+ extern int log_format_has_o_or_i;
+ extern int daemon_log_format_has_i;
+@@ -1002,7 +1003,7 @@ prepare_to_open:
+               statret = 0;
+       }
+-      if (dry_run || read_batch || whole_file)
++      if (!do_xfers || read_batch || whole_file)
+               goto notify_others;
+       if (fuzzy_basis) {
+@@ -1074,7 +1075,7 @@ notify_others:
+                       fuzzy_file ? fuzzy_file->basename : NULL);
+       }
+-      if (dry_run) {
++      if (!do_xfers) {
+               if (preserve_hard_links && file->link_u.links)
+                       hard_link_cluster(file, ndx, itemizing, code);
+               return;
+@@ -1122,9 +1123,9 @@ void generate_files(int f_out, struct fi
+               maybe_PERMS_REPORT = log_format_has_i ? 0 : PERMS_REPORT;
+               code = daemon_log_format_has_i ? 0 : FLOG;
+       } else if (am_daemon) {
+-              itemizing = daemon_log_format_has_i && !dry_run;
++              itemizing = daemon_log_format_has_i && do_xfers;
+               maybe_PERMS_REPORT = PERMS_REPORT;
+-              code = itemizing || dry_run ? FCLIENT : FINFO;
++              code = itemizing || !do_xfers ? FCLIENT : FINFO;
+       } else if (!am_server) {
+               itemizing = log_format_has_i;
+               maybe_PERMS_REPORT = log_format_has_i ? 0 : PERMS_REPORT;
+--- orig/main.c        2005-03-16 02:19:30
++++ main.c     2005-03-24 07:07:12
+@@ -168,7 +168,7 @@ static void report(int f)
+                       stats.flist_buildtime = read_longint(f);
+                       stats.flist_xfertime = read_longint(f);
+               }
+-      } else if (write_batch) {
++      } else if (write_batch && !am_server) {
+               /* The --read-batch process is going to be a client
+                * receiver, so we need to give it the stats. */
+               write_longint(batch_fd, total_read);
+@@ -583,7 +583,7 @@ static int do_recv(int f_in,int f_out,st
+       am_generator = 1;
+       close_multiplexing_in();
+-      if (write_batch)
++      if (write_batch && !am_server)
+               stop_write_batch();
+       close(error_pipe[1]);
+@@ -756,7 +756,7 @@ int client_run(int f_in, int f_out, pid_
+               if (filesfrom_host)
+                       filesfrom_fd = f_in;
+-              if (write_batch)
++              if (write_batch && !am_server)
+                       start_write_batch(f_out);
+               flist = send_file_list(f_out, argc, argv);
+               set_msg_fd_in(-1);
+@@ -793,7 +793,7 @@ int client_run(int f_in, int f_out, pid_
+               filesfrom_fd = -1;
+       }
+-      if (write_batch)
++      if (write_batch && !am_server)
+               start_write_batch(f_in);
+       flist = recv_file_list(f_in);
+       if (!flist || flist->count == 0) {
+@@ -1111,7 +1111,7 @@ int main(int argc,char *argv[])
+       init_flist();
+-      if (write_batch || read_batch) {
++      if ((write_batch || read_batch) && !am_server) {
+               if (write_batch)
+                       write_batch_shell_file(orig_argc, orig_argv, argc);
+@@ -1130,6 +1130,8 @@ int main(int argc,char *argv[])
+               if (read_batch)
+                       read_stream_flags(batch_fd);
+       }
++      if (write_batch < 0)
++              dry_run = 1;
+       if (am_daemon && !am_server)
+               return daemon_main();
+--- orig/options.c     2005-03-24 16:41:46
++++ options.c  2005-03-24 07:07:13
+@@ -53,6 +53,7 @@ int omit_dir_times = 0;
+ int update_only = 0;
+ int cvs_exclude = 0;
+ int dry_run = 0;
++int do_xfers = 1;
+ int ignore_times = 0;
+ int delete_mode = 0;
+ int delete_during = 0;
+@@ -344,6 +345,7 @@ void usage(enum logcode F)
+   rprintf(F,"     --list-only             list the files instead of copying them\n");
+   rprintf(F,"     --bwlimit=KBPS          limit I/O bandwidth; KBytes per second\n");
+   rprintf(F,"     --write-batch=FILE      write a batched update to FILE\n");
++  rprintf(F,"     --only-write-batch=FILE like --write-batch but w/o updating destination\n");
+   rprintf(F,"     --read-batch=FILE       read a batched update from FILE\n");
+ #ifdef INET6
+   rprintf(F," -4, --ipv4                  prefer IPv4\n");
+@@ -359,7 +361,8 @@ void usage(enum logcode F)
+ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
+       OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST,
+       OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
+-      OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT, OPT_MAX_SIZE,
++      OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH,
++      OPT_TIMEOUT, OPT_MAX_SIZE,
+       OPT_REFUSED_BASE = 9000};
+ static struct poptOption long_options[] = {
+@@ -449,6 +452,7 @@ static struct poptOption long_options[] 
+   {"hard-links",      'H', POPT_ARG_NONE,   &preserve_hard_links, 0, 0, 0 },
+   {"read-batch",       0,  POPT_ARG_STRING, &batch_name, OPT_READ_BATCH, 0, 0 },
+   {"write-batch",      0,  POPT_ARG_STRING, &batch_name, OPT_WRITE_BATCH, 0, 0 },
++  {"only-write-batch", 0,  POPT_ARG_STRING, &batch_name, OPT_ONLY_WRITE_BATCH, 0, 0 },
+   {"files-from",       0,  POPT_ARG_STRING, &files_from, 0, 0, 0 },
+   {"from0",           '0', POPT_ARG_NONE,   &eol_nulls, 0, 0, 0},
+   {"no-implied-dirs",  0,  POPT_ARG_VAL,    &implied_dirs, 0, 0, 0 },
+@@ -790,6 +794,11 @@ int parse_arguments(int *argc, const cha
+                       write_batch = 1;
+                       break;
++              case OPT_ONLY_WRITE_BATCH:
++                      /* batch_name is already set */
++                      write_batch = -1;
++                      break;
++
+               case OPT_READ_BATCH:
+                       /* batch_name is already set */
+                       read_batch = 1;
+@@ -900,7 +909,7 @@ int parse_arguments(int *argc, const cha
+                       "--write-batch and --read-batch can not be used together\n");
+               return 0;
+       }
+-      if (write_batch || read_batch) {
++      if (write_batch > 0 || read_batch) {
+               if (am_server) {
+                       rprintf(FINFO,
+                               "ignoring --%s-batch option sent to server\n",
+@@ -1092,6 +1101,9 @@ int parse_arguments(int *argc, const cha
+           && !am_server)
+               verbose = 1;
++      if (dry_run)
++              do_xfers = 0;
++
+       if (verbose && !log_format) {
+               log_format = "%n%L";
+               log_before_transfer = !am_server;
+@@ -1245,7 +1257,7 @@ void server_options(char **args,int *arg
+               argstr[x++] = 'b';
+       if (update_only)
+               argstr[x++] = 'u';
+-      if (dry_run)
++      if (!do_xfers)
+               argstr[x++] = 'n';
+       if (preserve_links)
+               argstr[x++] = 'l';
+@@ -1373,6 +1385,8 @@ void server_options(char **args,int *arg
+                       args[ac++] = "--delete-after";
+               if (force_delete)
+                       args[ac++] = "--force";
++              if (write_batch < 0)
++                      args[ac++] = "--only-write-batch=X";
+       }
+       if (size_only)
+--- orig/pipe.c        2005-02-07 20:36:43
++++ pipe.c     2005-03-24 07:07:13
+@@ -25,7 +25,6 @@ extern int am_sender;
+ extern int am_server;
+ extern int blocking_io;
+ extern int orig_umask;
+-extern int write_batch;
+ extern int filesfrom_fd;
+ /**
+@@ -128,10 +127,6 @@ pid_t local_child(int argc, char **argv,
+               am_sender = !am_sender;
+               am_server = 1;
+-              /* The server side never writes the batch, even if it
+-               * is local (it makes the logic easier elsewhere). */
+-              write_batch = 0;
+-
+               if (!am_sender)
+                       filesfrom_fd = -1;
+--- orig/receiver.c    2005-03-24 16:41:46
++++ receiver.c 2005-03-24 07:07:13
+@@ -21,7 +21,7 @@
+ #include "rsync.h"
+ extern int verbose;
+-extern int dry_run;
++extern int do_xfers;
+ extern int am_daemon;
+ extern int am_server;
+ extern int do_progress;
+@@ -30,6 +30,7 @@ extern int log_format_has_i;
+ extern int daemon_log_format_has_i;
+ extern int csum_length;
+ extern int read_batch;
++extern int write_batch;
+ extern int batch_gen_fd;
+ extern int protocol_version;
+ extern int relative_paths;
+@@ -435,13 +436,18 @@ int recv_files(int f_in, struct file_lis
+                       exit_cleanup(RERR_PROTOCOL);
+               }
+-              if (dry_run) { /* log the transfer */
++              if (!do_xfers) { /* log the transfer */
+                       if (!am_server && log_format)
+                               log_item(file, &stats, iflags, NULL);
+                       if (read_batch)
+                               discard_receive_data(f_in, file->length);
+                       continue;
+               }
++              if (write_batch < 0) {
++                      log_item(file, &stats, iflags, NULL);
++                      discard_receive_data(f_in, file->length);
++                      continue;
++              }
+               if (read_batch) {
+                       while (i > next_gen_i) {
+--- orig/rsync.yo      2005-03-16 02:19:30
++++ rsync.yo   2005-03-27 05:13:01
+@@ -379,6 +379,7 @@ to the detailed description below for a 
+      --list-only             list the files instead of copying them
+      --bwlimit=KBPS          limit I/O bandwidth; KBytes per second
+      --write-batch=FILE      write a batched update to FILE
++     --only-write-batch=FILE like --write-batch but w/o updating dest
+      --read-batch=FILE       read a batched update from FILE
+      --checksum-seed=NUM     set block/file checksum seed (advanced)
+  -4, --ipv4                  prefer IPv4
+@@ -1270,6 +1271,16 @@ dit(bf(--write-batch=FILE)) Record a fil
+ another identical destination with bf(--read-batch). See the "BATCH MODE"
+ section for details.
++dit(bf(--only-write-batch=FILE)) Works like bf(--write-batch), except that
++no updates are made on the destination system when creating the batch.
++This lets you transport the changes to the destination system via some
++other means and then apply the changes via bf(--read-batch).  Note that
++you can feel free to write the batch directly to some portable media: if
++this media fills to capacity before the end of the transfer, you can just
++apply that partial transfer to the destination and repeat the whole process
++to get the rest of the changes (as long as you don't mind a partially
++updated destination system while the multi-update cycle is happening).
++
+ dit(bf(--read-batch=FILE)) Apply all of the changes stored in FILE, a
+ file previously generated by bf(--write-batch).
+ If em(FILE) is "-" the batch data will be read from standard input.
+--- orig/sender.c      2005-03-16 02:19:30
++++ sender.c   2005-03-24 07:07:13
+@@ -20,7 +20,7 @@
+ #include "rsync.h"
+ extern int verbose;
+-extern int dry_run;
++extern int do_xfers;
+ extern int am_server;
+ extern int am_daemon;
+ extern int log_before_transfer;
+@@ -34,6 +34,8 @@ extern int updating_basis_file;
+ extern int make_backups;
+ extern int do_progress;
+ extern int inplace;
++extern int batch_fd;
++extern int write_batch;
+ extern struct stats stats;
+ extern struct file_list *the_file_list;
+ extern char *log_format;
+@@ -216,6 +218,7 @@ void send_files(struct file_list *flist,
+       int save_make_backups = make_backups;
+       int itemizing = am_daemon ? daemon_log_format_has_i
+                     : !am_server && log_format_has_i;
++      int f_xfer = write_batch < 0 ? batch_fd : f_out;
+       int i, j;
+       if (verbose > 2)
+@@ -238,7 +241,7 @@ void send_files(struct file_list *flist,
+                       continue;
+               }
+-              iflags = read_item_attrs(f_in, f_out, i, &fnamecmp_type,
++              iflags = read_item_attrs(f_in, f_xfer, i, &fnamecmp_type,
+                                        xname, &xlen);
+               if (iflags == ITEM_IS_NEW) /* no-op packet */
+                       continue;
+@@ -274,10 +277,10 @@ void send_files(struct file_list *flist,
+               stats.num_transferred_files++;
+               stats.total_transferred_size += file->length;
+-              if (dry_run) { /* log the transfer */
++              if (!do_xfers) { /* log the transfer */
+                       if (!am_server && log_format)
+                               log_item(file, &stats, iflags, NULL);
+-                      write_ndx_and_attrs(f_out, i, iflags, fnamecmp_type,
++                      write_ndx_and_attrs(f_xfer, i, iflags, fnamecmp_type,
+                                           xname, xlen);
+                       continue;
+               }
+@@ -329,9 +332,9 @@ void send_files(struct file_list *flist,
+                               safe_fname(fname), (double)st.st_size);
+               }
+-              write_ndx_and_attrs(f_out, i, iflags, fnamecmp_type,
++              write_ndx_and_attrs(f_xfer, i, iflags, fnamecmp_type,
+                                   xname, xlen);
+-              write_sum_head(f_out, s);
++              write_sum_head(f_xfer, s);
+               if (verbose > 2) {
+                       rprintf(FINFO, "calling match_sums %s\n",
+@@ -345,7 +348,7 @@ void send_files(struct file_list *flist,
+               set_compression(fname);
+-              match_sums(f_out, s, mbuf, st.st_size);
++              match_sums(f_xfer, s, mbuf, st.st_size);
+               if (do_progress)
+                       end_progress(st.st_size);