X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/blobdiff_plain/72f2d1b38495477d63142ed5a490164bff94bc90..d6081c829cf2aed6e7d5780e9a076fdcfd0e2391:/io.c

diff --git a/io.c b/io.c
index 6dafa2af..b4c552b7 100644
--- a/io.c
+++ b/io.c
@@ -105,6 +105,8 @@ static char io_filesfrom_lastchar;
 static int io_filesfrom_buflen;
 static size_t contiguous_write_len = 0;
 static int select_timeout = SELECT_TIMEOUT;
+static int active_filecnt = 0;
+static OFF_T active_bytecnt = 0;
 
 static void read_loop(int fd, char *buf, size_t len);
 
@@ -279,6 +281,8 @@ static void read_msg_fd(void)
 			exit_cleanup(RERR_STREAMIO);
 		}
 		read_loop(fd, buf, 4);
+		if (remove_sent_files)
+			decrement_active_files(IVAL(buf,0));
 		flist_ndx_push(&redo_list, IVAL(buf,0));
 		break;
 	case MSG_DELETED:
@@ -295,8 +299,10 @@ static void read_msg_fd(void)
 			exit_cleanup(RERR_STREAMIO);
 		}
 		read_loop(fd, buf, len);
-		if (remove_sent_files)
+		if (remove_sent_files) {
+			decrement_active_files(IVAL(buf,0));
 			io_multiplex_write(MSG_SUCCESS, buf, len);
+		}
 		if (preserve_hard_links)
 			flist_ndx_push(&hlink_list, IVAL(buf,0));
 		break;
@@ -327,6 +333,29 @@ static void read_msg_fd(void)
 	msg_fd_in = fd;
 }
 
+/* This is used by the generator to limit how many file transfers can
+ * be active at once when --remove-sent-files is specified.  Without
+ * this, sender-side deletions were mostly happening at the end. */
+void increment_active_files(int ndx, int itemizing, enum logcode code)
+{
+	/* TODO: tune these limits? */
+	while (active_filecnt >= 10
+	 && (active_bytecnt >= 128*1024 || active_filecnt >= 50)) {
+		if (hlink_list.head)
+			check_for_finished_hlinks(itemizing, code);
+		read_msg_fd();
+	}
+
+	active_filecnt++;
+	active_bytecnt += the_file_list->files[ndx]->length;
+}
+
+void decrement_active_files(int ndx)
+{
+	active_filecnt--;
+	active_bytecnt -= the_file_list->files[ndx]->length;
+}
+
 /* Try to push messages off the list onto the wire.  If we leave with more
  * to do, return 0.  On error, return -1.  If everything flushed, return 1.
  * This is only active in the receiver. */
@@ -774,12 +803,13 @@ static int readfd_unbuffered(int fd, char *buf, size_t len)
 			if (msg_bytes >= sizeof line)
 				goto overflow;
 			read_loop(fd, line, msg_bytes);
-			line[msg_bytes] = '\0';
 			/* A directory name was sent with the trailing null */
 			if (msg_bytes > 0 && !line[msg_bytes-1])
 				log_delete(line, S_IFDIR);
-			else
+			else {
+				line[msg_bytes] = '\0';
 				log_delete(line, S_IFREG);
+			}
 			break;
 		case MSG_SUCCESS:
 			if (msg_bytes != 4) {
@@ -1111,7 +1141,7 @@ static void writefd_unbuffered(int fd,char *buf,size_t len)
  **/
 static void mplex_write(enum msgcode code, char *buf, size_t len)
 {
-	char buffer[BIGPATHBUFLEN];
+	char buffer[1024];
 	size_t n = len;
 
 	SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len);
@@ -1124,9 +1154,10 @@ static void mplex_write(enum msgcode code, char *buf, size_t len)
 		contiguous_write_len = len + 4;
 
 	if (n > sizeof buffer - 4)
-		n = sizeof buffer - 4;
+		n = 0;
+	else
+		memcpy(buffer + 4, buf, n);
 
-	memcpy(&buffer[4], buf, n);
 	writefd_unbuffered(sock_f_out, buffer, n+4);
 
 	len -= n;