[rsync/rsync-patches.git] / trust-append.diff

This patch adds a --trust-append option that, in combination with --append,
makes rsync exclude the portion of a transferred file that was already on the
receiver from the post-transfer checksum.  In my limited testing, it seems to
work.

-- Matt McCutchen <hashproduct@gmail.com>

--- old/match.c
+++ new/match.c
@@ -25,6 +25,7 @@ extern int verbose;
 extern int do_progress;
 extern int checksum_seed;
 extern int append_mode;
+extern int trust_append;
 
 int updating_basis_file;
 
@@ -315,21 +316,23 @@ void match_sums(int f, struct sum_struct
 	sum_init(checksum_seed);
 
 	if (append_mode > 0) {
-		OFF_T j = 0;
-		for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
-			if (buf && do_progress)
-				show_progress(last_match, buf->file_size);
-			sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
-				   CHUNK_SIZE);
-			last_match = j;
-		}
-		if (last_match < s->flength) {
-			int32 n = (int32)(s->flength - last_match);
-			if (buf && do_progress)
-				show_progress(last_match, buf->file_size);
-			sum_update(map_ptr(buf, last_match, n), n);
-			last_match = s->flength;
+		if (!trust_append) {
+			OFF_T j = 0;
+			for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
+				if (buf && do_progress)
+					show_progress(last_match, buf->file_size);
+				sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
+					   CHUNK_SIZE);
+				last_match = j;
+			}
+			if (last_match < s->flength) {
+				int32 n = (int32)(s->flength - last_match);
+				if (buf && do_progress)
+					show_progress(last_match, buf->file_size);
+				sum_update(map_ptr(buf, last_match, n), n);
+			}
 		}
+		last_match = s->flength;
 		s->count = 0;
 	}
 
--- old/options.c
+++ new/options.c
@@ -41,6 +41,7 @@ int make_backups = 0;
 int whole_file = -1;
 
 int append_mode = 0;
+int trust_append = 0;
 int keep_dirlinks = 0;
 int copy_dirlinks = 0;
 int copy_links = 0;
@@ -313,6 +314,7 @@ void usage(enum logcode F)
   rprintf(F," -u, --update                skip files that are newer on the receiver\n");
   rprintf(F,"     --inplace               update destination files in-place (SEE MAN PAGE)\n");
   rprintf(F,"     --append                append data onto shorter files\n");
+  rprintf(F,"     --trust-append          ... and don't checksum portion already on dest\n");
   rprintf(F," -d, --dirs                  transfer directories without recursing\n");
   rprintf(F," -l, --links                 copy symlinks as symlinks\n");
   rprintf(F," -L, --copy-links            transform symlink into referent file/dir\n");
@@ -517,6 +519,7 @@ static struct poptOption long_options[] 
   {"sparse",          'S', POPT_ARG_NONE,   &sparse_files, 0, 0, 0 },
   {"inplace",          0,  POPT_ARG_NONE,   &inplace, 0, 0, 0 },
   {"append",           0,  POPT_ARG_VAL,    &append_mode, 1, 0, 0 },
+  {"trust-append",     0,  POPT_ARG_NONE,   &trust_append, 0, 0, 0 },
   {"del",              0,  POPT_ARG_NONE,   &delete_during, 0, 0, 0 },
   {"delete",           0,  POPT_ARG_NONE,   &delete_mode, 0, 0, 0 },
   {"delete-before",    0,  POPT_ARG_NONE,   &delete_before, 0, 0, 0 },
@@ -1898,9 +1901,11 @@ void server_options(char **args,int *arg
 		}
 	}
 
-	if (append_mode)
+	if (append_mode) {
 		args[ac++] = "--append";
-	else if (inplace)
+		if (trust_append)
+			args[ac++] = "--trust-append";
+	} else if (inplace)
 		args[ac++] = "--inplace";
 
 	if (files_from && (!am_sender || filesfrom_host)) {
--- old/receiver.c
+++ new/receiver.c
@@ -44,6 +44,7 @@ extern int make_backups;
 extern int cleanup_got_literal;
 extern int remove_source_files;
 extern int append_mode;
+extern int trust_append;
 extern int sparse_files;
 extern int keep_partial;
 extern int checksum_seed;
@@ -157,20 +158,22 @@ static int receive_data(int f_in, char *
 		sum.flength = (OFF_T)sum.count * sum.blength;
 		if (sum.remainder)
 			sum.flength -= sum.blength - sum.remainder;
-		for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
-			if (do_progress)
-				show_progress(offset, total_size);
-			sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
-				   CHUNK_SIZE);
-			offset = j;
-		}
-		if (offset < sum.flength) {
-			int32 len = (int32)(sum.flength - offset);
-			if (do_progress)
-				show_progress(offset, total_size);
-			sum_update(map_ptr(mapbuf, offset, len), len);
-			offset = sum.flength;
+		if (!trust_append) {
+			for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
+				if (do_progress)
+					show_progress(offset, total_size);
+				sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
+					   CHUNK_SIZE);
+				offset = j;
+			}
+			if (offset < sum.flength) {
+				int32 len = (int32)(sum.flength - offset);
+				if (do_progress)
+					show_progress(offset, total_size);
+				sum_update(map_ptr(mapbuf, offset, len), len);
+			}
 		}
+		offset = sum.flength;
 		if (fd != -1 && (j = do_lseek(fd, offset, SEEK_SET)) != offset) {
 			rsyserr(FERROR, errno, "lseek of %s returned %.0f, not %.0f",
 				full_fname(fname), (double)j, (double)offset);
--- old/rsync.yo
+++ new/rsync.yo
@@ -318,6 +318,7 @@ to the detailed description below for a 
  -u, --update                skip files that are newer on the receiver
      --inplace               update destination files in-place
      --append                append data onto shorter files
+     --trust-append          ... and don't checksum portion already on dest
  -d, --dirs                  transfer directories without recursing
  -l, --links                 copy symlinks as symlinks
  -L, --copy-links            transform symlink into referent file/dir
@@ -709,6 +710,12 @@ Implies bf(--inplace), but does not conf
 bf(--sparse) option will be auto-disabled if a resend of the already-existing
 data is required).
 
+dit(bf(--trust-append)) In combination with bf(--append), this option excludes
+the portion of a transferred file that was already on the receiver from the
+post-transfer checksum.  You can use this option to improve performance if you
+are certain that the receiver's file is identical to the start of the sender's
+file.
+
 dit(bf(-d, --dirs)) Tell the sending side to include any directories that
 are encountered.  Unlike bf(--recursive), a directory's contents are not copied
 unless the directory name specified is "." or ends with a trailing slash
Commit	Line	Data
24d21c7b WD	1	This patch adds a --trust-append option that, in combination with --append,
	2	makes rsync exclude the portion of a transferred file that was already on the
	3	receiver from the post-transfer checksum. In my limited testing, it seems to
	4	work.
e1d5c853 WD	5
	6	-- Matt McCutchen <hashproduct@gmail.com>
	7
	8	--- old/match.c
	9	+++ new/match.c
	10	@@ -25,6 +25,7 @@ extern int verbose;
	11	extern int do_progress;
	12	extern int checksum_seed;
	13	extern int append_mode;
	14	+extern int trust_append;
	15
	16	int updating_basis_file;
	17
	18	@@ -315,21 +316,23 @@ void match_sums(int f, struct sum_struct
	19	sum_init(checksum_seed);
	20
	21	if (append_mode > 0) {
	22	- OFF_T j = 0;
	23	- for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
	24	- if (buf && do_progress)
	25	- show_progress(last_match, buf->file_size);
	26	- sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
	27	- CHUNK_SIZE);
	28	- last_match = j;
	29	- }
	30	- if (last_match < s->flength) {
	31	- int32 n = (int32)(s->flength - last_match);
	32	- if (buf && do_progress)
	33	- show_progress(last_match, buf->file_size);
	34	- sum_update(map_ptr(buf, last_match, n), n);
	35	- last_match = s->flength;
	36	+ if (!trust_append) {
	37	+ OFF_T j = 0;
	38	+ for (j = CHUNK_SIZE; j < s->flength; j += CHUNK_SIZE) {
	39	+ if (buf && do_progress)
	40	+ show_progress(last_match, buf->file_size);
	41	+ sum_update(map_ptr(buf, last_match, CHUNK_SIZE),
	42	+ CHUNK_SIZE);
	43	+ last_match = j;
	44	+ }
	45	+ if (last_match < s->flength) {
	46	+ int32 n = (int32)(s->flength - last_match);
	47	+ if (buf && do_progress)
	48	+ show_progress(last_match, buf->file_size);
	49	+ sum_update(map_ptr(buf, last_match, n), n);
	50	+ }
	51	}
	52	+ last_match = s->flength;
	53	s->count = 0;
	54	}
	55
	56	--- old/options.c
	57	+++ new/options.c
	58	@@ -41,6 +41,7 @@ int make_backups = 0;
	59	int whole_file = -1;
	60
	61	int append_mode = 0;
	62	+int trust_append = 0;
	63	int keep_dirlinks = 0;
	64	int copy_dirlinks = 0;
	65	int copy_links = 0;
	66	@@ -313,6 +314,7 @@ void usage(enum logcode F)
	67	rprintf(F," -u, --update skip files that are newer on the receiver\n");
	68	rprintf(F," --inplace update destination files in-place (SEE MAN PAGE)\n");
69	rprintf(F," --append append data onto shorter files\n");
70	+ rprintf(F," --trust-append ... and don't checksum portion already on dest\n");
71	rprintf(F," -d, --dirs transfer directories without recursing\n");
72	rprintf(F," -l, --links copy symlinks as symlinks\n");
73	rprintf(F," -L, --copy-links transform symlink into referent file/dir\n");
74	@@ -517,6 +519,7 @@ static struct poptOption long_options[]
75	{"sparse", 'S', POPT_ARG_NONE, &sparse_files, 0, 0, 0 },
76	{"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 },
77	{"append", 0, POPT_ARG_VAL, &append_mode, 1, 0, 0 },
78	+ {"trust-append", 0, POPT_ARG_NONE, &trust_append, 0, 0, 0 },
79	{"del", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 },
80	{"delete", 0, POPT_ARG_NONE, &delete_mode, 0, 0, 0 },
81	{"delete-before", 0, POPT_ARG_NONE, &delete_before, 0, 0, 0 },
82	@@ -1898,9 +1901,11 @@ void server_options(char *args,int arg
83	}
84	}
85
86	- if (append_mode)
87	+ if (append_mode) {
88	args[ac++] = "--append";
89	- else if (inplace)
90	+ if (trust_append)
91	+ args[ac++] = "--trust-append";
92	+ } else if (inplace)
93	args[ac++] = "--inplace";
94
95	if (files_from && (!am_sender \|\| filesfrom_host)) {
96	--- old/receiver.c
97	+++ new/receiver.c
98	@@ -44,6 +44,7 @@ extern int make_backups;
99	extern int cleanup_got_literal;
100	extern int remove_source_files;
101	extern int append_mode;
102	+extern int trust_append;
103	extern int sparse_files;
104	extern int keep_partial;
105	extern int checksum_seed;
106	@@ -157,20 +158,22 @@ static int receive_data(int f_in, char *
107	sum.flength = (OFF_T)sum.count * sum.blength;
108	if (sum.remainder)
109	sum.flength -= sum.blength - sum.remainder;
110	- for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
111	- if (do_progress)
112	- show_progress(offset, total_size);
113	- sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
114	- CHUNK_SIZE);
115	- offset = j;
116	- }
117	- if (offset < sum.flength) {
118	- int32 len = (int32)(sum.flength - offset);
119	- if (do_progress)
120	- show_progress(offset, total_size);
121	- sum_update(map_ptr(mapbuf, offset, len), len);
122	- offset = sum.flength;
123	+ if (!trust_append) {
124	+ for (j = CHUNK_SIZE; j < sum.flength; j += CHUNK_SIZE) {
125	+ if (do_progress)
126	+ show_progress(offset, total_size);
127	+ sum_update(map_ptr(mapbuf, offset, CHUNK_SIZE),
128	+ CHUNK_SIZE);
129	+ offset = j;
130	+ }
131	+ if (offset < sum.flength) {
132	+ int32 len = (int32)(sum.flength - offset);
133	+ if (do_progress)
134	+ show_progress(offset, total_size);
135	+ sum_update(map_ptr(mapbuf, offset, len), len);
136	+ }
137	}
138	+ offset = sum.flength;
139	if (fd != -1 && (j = do_lseek(fd, offset, SEEK_SET)) != offset) {
140	rsyserr(FERROR, errno, "lseek of %s returned %.0f, not %.0f",
141	full_fname(fname), (double)j, (double)offset);
142	--- old/rsync.yo
143	+++ new/rsync.yo
144	@@ -318,6 +318,7 @@ to the detailed description below for a
145	-u, --update skip files that are newer on the receiver
146	--inplace update destination files in-place
147	--append append data onto shorter files
148	+ --trust-append ... and don't checksum portion already on dest
149	-d, --dirs transfer directories without recursing
150	-l, --links copy symlinks as symlinks
151	-L, --copy-links transform symlink into referent file/dir
152	@@ -709,6 +710,12 @@ Implies bf(--inplace), but does not conf
153	bf(--sparse) option will be auto-disabled if a resend of the already-existing
154	data is required).
155
156	+dit(bf(--trust-append)) In combination with bf(--append), this option excludes
157	+the portion of a transferred file that was already on the receiver from the
158	+post-transfer checksum. You can use this option to improve performance if you
159	+are certain that the receiver's file is identical to the start of the sender's
160	+file.
161	+
162	dit(bf(-d, --dirs)) Tell the sending side to include any directories that
163	are encountered. Unlike bf(--recursive), a directory's contents are not copied
164	unless the directory name specified is "." or ends with a trailing slash