A new option: --append.

[rsync/rsync-patches.git] / append.diff

This patch adds the --append option, which works like a "resume" mode in
an ftp client, appending new data onto the end of the files it updates.

--- orig/generator.c	2005-02-11 20:25:07
+++ generator.c	2005-02-11 20:26:31
@@ -40,6 +40,7 @@ extern int delete_during;
 extern int update_only;
 extern int opt_ignore_existing;
 extern int inplace;
+extern int append_mode;
 extern int make_backups;
 extern int csum_length;
 extern int ignore_times;
@@ -192,35 +193,42 @@ static void generate_and_send_sums(int f
 	OFF_T offset = 0;
 
 	sum_sizes_sqroot(&sum, len);
+	write_sum_head(f_out, &sum);
+
+	if (append_mode > 0 && f_copy < 0)
+		return;
 
 	if (len > 0)
 		mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
 	else
 		mapbuf = NULL;
 
-	write_sum_head(f_out, &sum);
-
 	for (i = 0; i < sum.count; i++) {
 		int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
 		char *map = map_ptr(mapbuf, offset, n1);
-		uint32 sum1 = get_checksum1(map, n1);
 		char sum2[SUM_LENGTH];
+		uint32 sum1;
+
+		len -= n1;
+		offset += n1;
 
-		if (f_copy >= 0)
+		if (f_copy >= 0) {
 			full_write(f_copy, map, n1);
+			if (append_mode > 0)
+				continue;
+		}
 
+		sum1 = get_checksum1(map, n1);
 		get_checksum2(map, n1, sum2);
 
 		if (verbose > 3) {
 			rprintf(FINFO,
 				"chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
-				(double)i, (double)offset, (long)n1,
+				(double)i, (double)offset - n1, (long)n1,
 				(unsigned long)sum1);
 		}
 		write_int(f_out, sum1);
 		write_buf(f_out, sum2, sum.s2length);
-		len -= n1;
-		offset += n1;
 	}
 
 	if (mapbuf)
@@ -509,6 +517,9 @@ static void recv_generator(char *fname, 
 		return;
 	}
 
+	if (append_mode && st.st_size > file->length)
+		return;
+
 	if (!compare_dest && fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
 		;
 	else if (unchanged_file(fnamecmp, file, &st)) {
@@ -527,7 +538,7 @@ prepare_to_open:
 
 	if (dry_run || read_batch)
 		goto notify_others;
-	if (whole_file > 0) {
+	if (whole_file > 0 && !append_mode) {
 		statret = -1;
 		goto notify_others;
 	}
@@ -664,6 +675,9 @@ void generate_files(int f_out, struct fi
 	csum_length = SUM_LENGTH;
 	only_existing = max_size = opt_ignore_existing = 0;
 	update_only = always_checksum = size_only = 0;
+	make_backups = 0; /* avoid a duplicate backup in inplace mode */
+	if (append_mode)  /* resend w/o append mode */
+		append_mode = -1; /* ... but only longer files */
 	ignore_times = 1;
 
 	if (verbose > 2)
--- orig/match.c	2005-01-17 23:11:45
+++ match.c	2005-02-11 20:26:31
@@ -23,6 +23,7 @@ extern int verbose;
 extern int am_server;
 extern int do_progress;
 extern int checksum_seed;
+extern int append_mode;
 
 int updating_basis_file;
 
@@ -336,6 +337,21 @@ void match_sums(int f, struct sum_struct
 
 	sum_init(checksum_seed);
 
+	if (append_mode) {
+		OFF_T j = 0;
+		for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
+			sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
+				   CHUNK_SIZE);
+			last_match = j;
+		}
+		if (last_match < s->flength) {
+			int32 len = s->flength - last_match;
+			sum_update(map_ptr(buf, last_match, len), len);
+			last_match = s->flength;
+		}
+		s->count = 0;
+	}
+
 	if (len > 0 && s->count > 0) {
 		build_hash_table(s);
 
@@ -349,7 +365,7 @@ void match_sums(int f, struct sum_struct
 	} else {
 		OFF_T j;
 		/* by doing this in pieces we avoid too many seeks */
-		for (j = CHUNK_SIZE; j < len; j += CHUNK_SIZE)
+		for (j = last_match + CHUNK_SIZE; j < len; j += CHUNK_SIZE)
 			matched(f, s, buf, j, -2);
 		matched(f, s, buf, len, -1);
 	}
--- orig/options.c	2005-02-11 18:21:45
+++ options.c	2005-02-11 20:31:32
@@ -39,6 +39,7 @@ int make_backups = 0;
  **/
 int whole_file = -1;
 
+int append_mode = 0;
 int archive_mode = 0;
 int keep_dirlinks = 0;
 int copy_links = 0;
@@ -150,7 +151,7 @@ static int daemon_opt;   /* sets am_daem
 static int F_option_cnt = 0;
 static int modify_window_set;
 static int refused_verbose, refused_delete, refused_archive_part;
-static int refused_partial, refused_progress;
+static int refused_partial, refused_progress, refused_inplace;
 static char *dest_option = NULL;
 static char *max_size_arg;
 static char partialdir_for_delayupdate[] = ".~tmp~";
@@ -260,6 +261,7 @@ void usage(enum logcode F)
   rprintf(F,"     --suffix=SUFFIX         set backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
   rprintf(F," -u, --update                skip files that are newer on the receiver\n");
   rprintf(F,"     --inplace               update destination files in-place (SEE MAN PAGE)\n");
+  rprintf(F,"     --append                append data onto shorter files\n");
   rprintf(F," -d, --dirs                  transfer directories without recursing\n");
   rprintf(F," -l, --links                 copy symlinks as symlinks\n");
   rprintf(F," -L, --copy-links            transform symlink into referent file/dir\n");
@@ -383,6 +385,7 @@ static struct poptOption long_options[] 
   {"links",           'l', POPT_ARG_NONE,   &preserve_links, 0, 0, 0 },
   {"copy-links",      'L', POPT_ARG_NONE,   &copy_links, 0, 0, 0 },
   {"keep-dirlinks",   'K', POPT_ARG_NONE,   &keep_dirlinks, 0, 0, 0 },
+  {"append",           0,  POPT_ARG_VAL,    &append_mode, 1, 0, 0 },
   {"whole-file",      'W', POPT_ARG_VAL,    &whole_file, 1, 0, 0 },
   {"no-whole-file",    0,  POPT_ARG_VAL,    &whole_file, 0, 0, 0 },
   {"copy-unsafe-links", 0, POPT_ARG_NONE,   &copy_unsafe_links, 0, 0, 0 },
@@ -555,6 +558,8 @@ static void set_refuse_options(char *bp)
 						refused_partial = op->val;
 					else if (wildmatch("progress", op->longName))
 						refused_progress = op->val;
+					else if (wildmatch("inplace", op->longName))
+						refused_inplace = op->val;
 					break;
 				}
 				if (!is_wild)
@@ -1060,6 +1065,14 @@ int parse_arguments(int *argc, const cha
 			bwlimit_writemax = 512;
 	}
 
+	if (append_mode) {
+		if (refused_inplace) {
+			create_refuse_error(refused_inplace);
+			return 0;
+		}
+		inplace = 1;
+	}
+
 	if (delay_updates && !partial_dir)
 		partial_dir = partialdir_for_delayupdate;
 
@@ -1346,7 +1359,9 @@ void server_options(char **args,int *arg
 	if (opt_ignore_existing && am_sender)
 		args[ac++] = "--ignore-existing";
 
-	if (inplace)
+	if (append_mode)
+		args[ac++] = "--append";
+	else if (inplace)
 		args[ac++] = "--inplace";
 
 	if (tmpdir) {
--- orig/receiver.c	2005-02-11 10:53:14
+++ receiver.c	2005-02-11 20:26:32
@@ -44,6 +44,7 @@ extern int cleanup_got_literal;
 extern int module_id;
 extern int ignore_errors;
 extern int orig_umask;
+extern int append_mode;
 extern int keep_partial;
 extern int checksum_seed;
 extern int inplace;
@@ -164,6 +165,28 @@ static int receive_data(int f_in, char *
 
 	sum_init(checksum_seed);
 
+	if (append_mode) {
+		OFF_T j;
+		sum.flength = (OFF_T)sum.count * sum.blength;
+		if (sum.remainder)
+			sum.flength -= sum.blength - sum.remainder;
+		for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
+			sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
+				   CHUNK_SIZE);
+			offset = j;
+		}
+		if (offset < sum.flength) {
+			int32 len = sum.flength - offset;
+			sum_update(map_ptr(mapbuf, offset, len), len);
+			offset = sum.flength;
+		}
+		if (fd != -1 && do_lseek(fd, offset, SEEK_SET) != offset) {
+			rsyserr(FERROR, errno, "lseek failed on %s",
+				full_fname(fname));
+			exit_cleanup(RERR_FILEIO);
+		}
+	}
+
 	while ((i = recv_token(f_in, &data)) != 0) {
 		if (do_progress)
 			show_progress(offset, total_size);
@@ -325,6 +348,7 @@ int recv_files(int f_in, struct file_lis
 			send_msg(MSG_DONE, "", 0);
 			if (keep_partial && !partial_dir)
 				make_backups = 0; /* prevents double backup */
+			append_mode = 0;
 			continue;
 		}
 
--- orig/rsync.yo	2005-02-11 10:53:15
+++ rsync.yo	2005-02-11 20:36:38
@@ -309,6 +309,7 @@ to the detailed description below for a 
      --suffix=SUFFIX         backup suffix (default ~ w/o --backup-dir)
  -u, --update                skip files that are newer on the receiver
      --inplace               update destination files in-place
+     --append                append data onto shorter files
  -d, --dirs                  transfer directories without recursing
  -l, --links                 copy symlinks as symlinks
  -L, --copy-links            transform symlink into referent file/dir
@@ -553,6 +554,14 @@ should not use this option to update fil
 rsync will be unable to update a file in-place that is not writable by the
 receiving user.
 
+dit(bf(--append)) This causes rsync to update a file by appending data onto
+the end of the file, which presumes that the data that already exists on
+the receiving side is identical with the start of the file on the sending
+side.  If that is not true, the file will fail the checksum check, and a
+normal bf(--inplace) update will correct the mismatch.  Any file on the
+receiving side that is longer than a file on the sending side is skipped.
+Implies bf(--inplace).
+
 dit(bf(-d, --dirs)) Tell the sending side to include any directories that
 are encountered.  Unlike bf(--recursive), a directory's contents are not copied
 unless the directory was specified on the command-line as either "." or a
--- orig/sender.c	2005-02-03 02:04:20
+++ sender.c	2005-02-11 20:26:32
@@ -30,6 +30,7 @@ extern int protocol_version;
 extern int updating_basis_file;
 extern int make_backups;
 extern int inplace;
+extern int append_mode;
 extern struct stats stats;
 
 
@@ -62,6 +63,13 @@ static struct sum_struct *receive_sums(i
 			(double)s->count, (long)s->blength, (long)s->remainder);
 	}
 
+	if (append_mode) {
+		s->flength = (OFF_T)s->count * s->blength;
+		if (s->remainder)
+			s->flength -= s->blength - s->remainder;
+		return s;
+	}
+
 	if (s->count == 0)
 		return(s);
 
@@ -127,6 +135,7 @@ void send_files(struct file_list *flist,
 				/* For inplace: redo phase turns off the backup
 				 * flag so that we do a regular inplace send. */
 				make_backups = 0;
+				append_mode = 0;
 				continue;
 			}
 			break;