From ce827c3e50cd168027a555ec961b6636e5b63e59 Mon Sep 17 00:00:00 2001
From: Wayne Davison <wayned@samba.org>
Date: Sat, 22 Aug 2009 08:15:26 -0700
Subject: [PATCH] Have the sender use dead time to pad out the file list.

---
 flist.c     | 11 ++++++-----
 generator.c | 10 +++++++---
 io.c        | 14 ++++++++++++--
 rsync.c     | 11 ++++++++++-
 rsync.h     |  3 ++-
 sender.c    |  4 ++--
 6 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/flist.c b/flist.c
index 7208efa4..6eb02c7b 100644
--- a/flist.c
+++ b/flist.c
@@ -97,6 +97,7 @@ struct file_list *cur_flist, *first_flist, *dir_flist;
 int send_dir_ndx = -1, send_dir_depth = -1;
 int flist_cnt = 0; /* how many (non-tmp) file list objects exist */
 int file_total = 0; /* total of all active items over all file-lists */
+int file_old_total = 0; /* total of active items that will soon be gone */
 int flist_eof = 0; /* all the file-lists are now known */
 
 #define NORMAL_NAME 0
@@ -1905,17 +1906,17 @@ void send_extra_file_list(int f, int at_least)
 	struct file_list *flist;
 	int64 start_write;
 	uint16 prev_flags;
-	int old_cnt, save_io_error = io_error;
+	int save_io_error = io_error;
 
 	if (flist_eof)
 		return;
 
+	if (at_least < 0)
+		at_least = file_total - file_old_total + 1;
+
 	/* Keep sending data until we have the requested number of
 	 * files in the upcoming file-lists. */
-	old_cnt = cur_flist->used;
-	for (flist = first_flist; flist != cur_flist; flist = flist->next)
-		old_cnt += flist->used;
-	while (file_total - old_cnt < at_least) {
+	while (file_total - file_old_total < at_least) {
 		struct file_struct *file = dir_flist->sorted[send_dir_ndx];
 		int dir_ndx, dstart = stats.num_dirs;
 		const char *pathname = F_PATHNAME(file);
diff --git a/generator.c b/generator.c
index 99b80576..6f4fa4f0 100644
--- a/generator.c
+++ b/generator.c
@@ -1949,8 +1949,12 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 			break;
 
 		write_ndx(sock_f_out, NDX_DONE);
-		if (!read_batch)
-			maybe_flush_socket(1);
+		if (!read_batch && !flist_eof) {
+			int old_total = 0;
+			for (flist = first_flist; flist != cur_flist; flist = flist->next)
+				old_total += flist->used;
+			maybe_flush_socket(!flist_eof && file_total - old_total < MIN_FILECNT_LOOKAHEAD/2);
+		}
 
 		if (delete_during == 2 || !dir_tweaking) {
 			/* Skip directory touch-up. */
@@ -2028,7 +2032,7 @@ void generate_files(int f_out, const char *local_name)
 	do {
 #ifdef SUPPORT_HARD_LINKS
 		if (preserve_hard_links && inc_recurse) {
-			while (!flist_eof && file_total < FILECNT_LOOKAHEAD/2)
+			while (!flist_eof && file_total < MIN_FILECNT_LOOKAHEAD/2)
 				wait_for_receiver();
 		}
 #endif
diff --git a/io.c b/io.c
index 6a89c8fa..c9d990ad 100644
--- a/io.c
+++ b/io.c
@@ -45,6 +45,8 @@ extern int inc_recurse;
 extern int io_error;
 extern int eol_nulls;
 extern int flist_eof;
+extern int file_total;
+extern int file_old_total;
 extern int list_only;
 extern int read_batch;
 extern int protect_args;
@@ -645,7 +647,12 @@ static int read_timeout(int fd, char *buf, size_t len)
 				maxfd = new_fd;
 		}
 
-		tv.tv_sec = select_timeout;
+		if (am_sender && inc_recurse && !flist_eof && !defer_forwarding_messages && !cnt
+		 && file_total - file_old_total < MAX_FILECNT_LOOKAHEAD
+		 && file_total - file_old_total >= MIN_FILECNT_LOOKAHEAD)
+			tv.tv_sec = 0;
+		else
+			tv.tv_sec = select_timeout;
 		tv.tv_usec = 0;
 
 		errno = 0;
@@ -657,7 +664,10 @@ static int read_timeout(int fd, char *buf, size_t len)
 				defer_forwarding_messages = 0;
 				exit_cleanup(RERR_SOCKETIO);
 			}
-			check_timeout();
+			if (am_sender && tv.tv_sec == 0)
+				send_extra_file_list(sock_f_out, -1);
+			else
+				check_timeout();
 			continue;
 		}
 
diff --git a/rsync.c b/rsync.c
index 2c026a2d..d8bc469e 100644
--- a/rsync.c
+++ b/rsync.c
@@ -45,6 +45,7 @@ extern int gid_ndx;
 extern int inc_recurse;
 extern int inplace;
 extern int flist_eof;
+extern int file_old_total;
 extern int msgs2stderr;
 extern int keep_dirlinks;
 extern int make_backups;
@@ -313,7 +314,15 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 		goto read_loop;
 	}
 
-	cur_flist = flist_for_ndx(ndx, "read_ndx_and_attrs");
+	flist = flist_for_ndx(ndx, "read_ndx_and_attrs");
+	if (flist != cur_flist) {
+		cur_flist = flist;
+		if (am_sender) {
+			file_old_total = cur_flist->used;
+			for (flist = first_flist; flist != cur_flist; flist = flist->next)
+				file_old_total += flist->used;
+		}
+	}
 
 	if (iflags & ITEM_BASIS_TYPE_FOLLOWS)
 		fnamecmp_type = read_byte(f_in);
diff --git a/rsync.h b/rsync.h
index 4a15c24a..c8115fce 100644
--- a/rsync.h
+++ b/rsync.h
@@ -119,7 +119,8 @@
 #define OLD_PROTOCOL_VERSION 25
 #define MAX_PROTOCOL_VERSION 40
 
-#define FILECNT_LOOKAHEAD 1000
+#define MIN_FILECNT_LOOKAHEAD 1000
+#define MAX_FILECNT_LOOKAHEAD 10000
 
 #define RSYNC_PORT 873
 
diff --git a/sender.c b/sender.c
index cc2c9e90..815371a0 100644
--- a/sender.c
+++ b/sender.c
@@ -179,7 +179,7 @@ void send_files(int f_in, int f_out)
 
 	while (1) {
 		if (inc_recurse)
-			send_extra_file_list(f_out, FILECNT_LOOKAHEAD);
+			send_extra_file_list(f_out, MIN_FILECNT_LOOKAHEAD);
 
 		/* This call also sets cur_flist. */
 		ndx = read_ndx_and_attrs(f_in, &iflags, &fnamecmp_type,
@@ -205,7 +205,7 @@ void send_files(int f_in, int f_out)
 		}
 
 		if (inc_recurse)
-			send_extra_file_list(f_out, FILECNT_LOOKAHEAD);
+			send_extra_file_list(f_out, MIN_FILECNT_LOOKAHEAD);
 
 		if (ndx - cur_flist->ndx_start >= 0)
 			file = cur_flist->files[ndx - cur_flist->ndx_start];
-- 
2.34.1