X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/blobdiff_plain/1faa1a6da93065febb7e08830e5da37a09b6444d..abdcb21a7aa725cbf7a5a9e37f93484a133b69d1:/flist.c

diff --git a/flist.c b/flist.c
index 3982662f..e6a80853 100644
--- a/flist.c
+++ b/flist.c
@@ -38,6 +38,7 @@ extern int module_id;
 extern int ignore_errors;
 extern int numeric_ids;
 extern int recurse;
+extern int use_qsort;
 extern int xfer_dirs;
 extern int filesfrom_fd;
 extern int one_file_system;
@@ -76,6 +77,8 @@ extern int need_unsorted_flist;
 extern iconv_t ic_send, ic_recv;
 #endif
 
+#define PTR_SIZE (sizeof (struct file_struct *))
+
 int io_error;
 int checksum_len;
 dev_t filesystem_dev; /* used to implement -x */
@@ -305,6 +308,15 @@ static void flist_expand(struct file_list *flist, int extra)
 		out_of_memory("flist_expand");
 }
 
+static void flist_done_allocating(struct file_list *flist)
+{
+	void *ptr = pool_boundary(flist->file_pool, 8*1024);
+	if (flist->pool_boundary == ptr)
+		flist->pool_boundary = NULL; /* list didn't use any pool memory */
+	else
+		flist->pool_boundary = ptr;
+}
+
 int push_pathname(const char *dir, int len)
 {
 	if (dir == pathname)
@@ -1309,9 +1321,70 @@ static void send_if_directory(int f, struct file_list *flist,
 	}
 }
 
-static int file_compare(struct file_struct **file1, struct file_struct **file2)
+static int file_compare(const void *file1, const void *file2)
+{
+	return f_name_cmp(*(struct file_struct **)file1,
+			  *(struct file_struct **)file2);
+}
+
+/* The guts of a merge-sort algorithm.  This was derived from the glibc
+ * version, but I (Wayne) changed the merge code to do less copying and
+ * to require only half the amount of temporary memory. */
+static void fsort_tmp(struct file_struct **fp, size_t num,
+		      struct file_struct **tmp)
+{
+	struct file_struct **f1, **f2, **t;
+	size_t n1, n2;
+
+	n1 = num / 2;
+	n2 = num - n1;
+	f1 = fp;
+	f2 = fp + n1;
+
+	if (n1 > 1)
+		fsort_tmp(f1, n1, tmp);
+	if (n2 > 1)
+		fsort_tmp(f2, n2, tmp);
+
+	while (f_name_cmp(*f1, *f2) <= 0) {
+		if (!--n1)
+			return;
+		f1++;
+	}
+
+	t = tmp;
+	memcpy(t, f1, n1 * PTR_SIZE);
+
+	*f1++ = *f2++, n2--;
+
+	while (n1 > 0 && n2 > 0) {
+		if (f_name_cmp(*t, *f2) <= 0)
+			*f1++ = *t++, n1--;
+		else
+			*f1++ = *f2++, n2--;
+	}
+
+	if (n1 > 0)
+		memcpy(f1, t, n1 * PTR_SIZE);
+}
+
+/* This file-struct sorting routine makes sure that any identical names in
+ * the file list stay in the same order as they were in the original list.
+ * This is particularly vital in inc_recurse mode where we expect a sort
+ * on the flist to match the exact order of a sort on the dir_flist. */
+static void fsort(struct file_struct **fp, size_t num)
 {
-	return f_name_cmp(*file1, *file2);
+	if (num <= 1)
+		return;
+
+	if (use_qsort)
+		qsort(fp, num, PTR_SIZE, file_compare);
+	else {
+		struct file_struct **tmp = new_array(struct file_struct *,
+						     (num+1) / 2);
+		fsort_tmp(fp, num, tmp);
+		free(tmp);
+	}
 }
 
 /* We take an entire set of sibling dirs from the sorted flist and link them
@@ -1455,55 +1528,34 @@ void send_extra_file_list(int f, int at_least)
 		future_cnt = 0;
 	while (future_cnt < at_least) {
 		struct file_struct *file = dir_flist->sorted[send_dir_ndx];
-		int dstart = dir_count;
+		int dir_ndx, dstart = dir_count;
 		int32 *dp;
 
 		flist = flist_new(0, "send_extra_file_list");
 		start_write = stats.total_written;
 
-		/* If this is the first of a set of duplicate dirs, we must
-		 * send all the dirs together in a single file-list.  We must
-		 * also send the index of the last dir in the header. */
-		if (file->flags & FLAG_DUPLICATE) {
-			int dir_ndx, end_ndx = send_dir_ndx;
-			struct file_struct *fp = file;
-
-			while (1) {
-				dp = F_DIRNODE_P(fp);
-				end_ndx = DIR_NEXT_SIBLING(dp);
-				fp = dir_flist->sorted[end_ndx];
-				if (!(fp->flags & FLAG_DUPLICATE))
-					break;
-			}
-
 #ifdef ICONV_OPTION
-			if (ic_ndx)
-				dir_ndx = F_NDX(fp);
-			else
+		if (ic_ndx)
+			dir_ndx = F_NDX(file);
+		else
 #endif
-				dir_ndx = end_ndx;
-			write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
+			dir_ndx = send_dir_ndx;
+		write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
 
-			while (1) {
-				send1extra(f, file, flist);
-				if (send_dir_ndx == end_ndx)
-					break;
-				dp = F_DIRNODE_P(file);
-				send_dir_ndx = DIR_NEXT_SIBLING(dp);
-				file = dir_flist->sorted[send_dir_ndx];
-			}
-		} else {
-			int dir_ndx;
-#ifdef ICONV_OPTION
-			if (ic_ndx)
-				dir_ndx = F_NDX(file);
-			else
-#endif
-				dir_ndx = send_dir_ndx;
-			write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
+		send1extra(f, file, flist);
+		dp = F_DIRNODE_P(file);
 
+		/* If there are any duplicate directory names that follow, we
+		 * send all the dirs together in one file-list.  The dir_flist
+		 * tree links all the child subdirs onto the last dup dir. */
+		while ((dir_ndx = DIR_NEXT_SIBLING(dp)) >= 0
+		    && dir_flist->sorted[dir_ndx]->flags & FLAG_DUPLICATE) {
+			send_dir_ndx = dir_ndx;
+			file = dir_flist->sorted[dir_ndx];
 			send1extra(f, file, flist);
+			dp = F_DIRNODE_P(file);
 		}
+
 		write_byte(f, 0);
 
 #ifdef ICONV_OPTION
@@ -1519,6 +1571,7 @@ void send_extra_file_list(int f, int at_least)
 		clean_flist(flist, 0);
 
 		add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart);
+		flist_done_allocating(flist);
 
 		file_total += flist->count;
 		future_cnt += flist->count;
@@ -1527,7 +1580,6 @@ void send_extra_file_list(int f, int at_least)
 		if (verbose > 3)
 			output_flist(flist);
 
-		dp = F_DIRNODE_P(file);
 		if (DIR_FIRST_CHILD(dp) >= 0) {
 			send_dir_ndx = DIR_FIRST_CHILD(dp);
 			send_dir_depth++;
@@ -1853,6 +1905,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
 	if (inc_recurse) {
 		add_dirs_to_tree(-1, flist, dir_count);
+		flist_done_allocating(flist);
 		if (send_dir_ndx < 0) {
 			write_ndx(f, NDX_FLIST_EOF);
 			flist_eof = 1;
@@ -1947,8 +2000,7 @@ struct file_list *recv_file_list(int f)
 						dir_flist->count);
 			memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
 			       (dir_flist->count - dstart) * sizeof (struct file_struct*));
-			qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
-			      sizeof (struct file_struct*), (int (*)())file_compare);
+			fsort(dir_flist->sorted + dstart, dir_flist->count - dstart);
 		}
 	} else
 #endif
@@ -1956,12 +2008,13 @@ struct file_list *recv_file_list(int f)
 		flist->sorted = flist->files;
 		if (inc_recurse && dir_flist->count > dstart) {
 			dir_flist->sorted = dir_flist->files;
-			qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
-			      sizeof (struct file_struct*), (int (*)())file_compare);
+			fsort(dir_flist->sorted + dstart, dir_flist->count - dstart);
 		}
 	}
 
-	if (!inc_recurse && f >= 0)
+	if (inc_recurse)
+		flist_done_allocating(flist);
+	else if (f >= 0)
 		recv_id_list(f, flist);
 
 	clean_flist(flist, relative_paths);
@@ -2097,34 +2150,42 @@ struct file_list *flist_new(int flags, char *msg)
 
 	memset(flist, 0, sizeof flist[0]);
 
-	if (!(flags & FLIST_TEMP)) {
-		if (first_flist) {
-			flist->ndx_start = first_flist->prev->ndx_start
-					 + first_flist->prev->count;
-		}
+	if (flags & FLIST_TEMP) {
+		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,
+						out_of_memory, POOL_INTERN)))
+			out_of_memory(msg);
+	} else {
 		/* This is a doubly linked list with prev looping back to
 		 * the end of the list, but the last next pointer is NULL. */
-		if (!first_flist)
+		if (!first_flist) {
+			flist->file_pool = pool_create(NORMAL_EXTENT, 0,
+						out_of_memory, POOL_INTERN);
+			if (!flist->file_pool)
+				out_of_memory(msg);
+
 			first_flist = cur_flist = flist->prev = flist;
-		else {
+		} else {
+			flist->file_pool = first_flist->file_pool;
+
+			flist->ndx_start = first_flist->prev->ndx_start
+					 + first_flist->prev->count;
+
 			flist->prev = first_flist->prev;
 			flist->prev->next = first_flist->prev = flist;
 		}
+		flist->pool_boundary = pool_boundary(flist->file_pool, 0);
 		flist_cnt++;
 	}
 
-	if (!(flist->file_pool = pool_create(FILE_EXTENT, 0, out_of_memory, POOL_INTERN)))
-		out_of_memory(msg);
-
 	return flist;
 }
 
 /* Free up all elements in a flist. */
 void flist_free(struct file_list *flist)
 {
-	if (!flist->prev)
-		; /* Was FLIST_TEMP dir-list. */
-	else if (flist == flist->prev) {
+	if (!flist->prev) {
+		/* Was FLIST_TEMP dir-list. */
+	} else if (flist == flist->prev) {
 		first_flist = cur_flist = NULL;
 		file_total = 0;
 		flist_cnt = 0;
@@ -2143,7 +2204,11 @@ void flist_free(struct file_list *flist)
 		flist_cnt--;
 	}
 
-	pool_destroy(flist->file_pool);
+	if (!flist->prev || !flist_cnt)
+		pool_destroy(flist->file_pool);
+	else
+		pool_free_old(flist->file_pool, flist->pool_boundary);
+
 	if (flist->sorted && flist->sorted != flist->files)
 		free(flist->sorted);
 	free(flist->files);
@@ -2164,8 +2229,7 @@ static void clean_flist(struct file_list *flist, int strip_root)
 		return;
 	}
 
-	qsort(flist->sorted, flist->count,
-	    sizeof flist->sorted[0], (int (*)())file_compare);
+	fsort(flist->sorted, flist->count);
 
 	if (!am_sender || inc_recurse) {
 		for (i = prev_i = 0; i < flist->count; i++) {
@@ -2199,20 +2263,18 @@ static void clean_flist(struct file_list *flist, int strip_root)
 		} else
 			j = -1;
 		if (j >= 0) {
-			struct file_struct *fp = flist->sorted[j];
 			int keep, drop;
 			/* If one is a dir and the other is not, we want to
 			 * keep the dir because it might have contents in the
 			 * list. */
-			if (S_ISDIR(file->mode) != S_ISDIR(fp->mode)) {
-				if (S_ISDIR(file->mode))
+			if (S_ISDIR(file->mode)) {
+				struct file_struct *fp = flist->sorted[j];
+				if (!S_ISDIR(fp->mode))
 					keep = i, drop = j;
 				else
 					keep = j, drop = i;
-			} else if (protocol_version < 27)
+			} else
 				keep = j, drop = i;
-			else
-				keep = i, drop = j;
 
 			if (am_sender)
 				flist->sorted[drop]->flags |= FLAG_DUPLICATE;