X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/blobdiff_plain/8ea07c0030ee9dd2c56d4c8c57139887c0c6794e..b3b326016a4d29fea20d97e782f1af05542aca22:/flist.c

diff --git a/flist.c b/flist.c
index d7f78f16..a6c324ae 100644
--- a/flist.c
+++ b/flist.c
@@ -7,7 +7,7 @@
  * Copyright (C) 2002-2007 Wayne Davison
  *
  * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
+ * it under the terms of the GNU General Public License version 3 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
@@ -16,8 +16,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
+ * with this program; if not, visit the http://fsf.org website.
  */
 
 #include "rsync.h"
@@ -38,6 +37,7 @@ extern int module_id;
 extern int ignore_errors;
 extern int numeric_ids;
 extern int recurse;
+extern int use_qsort;
 extern int xfer_dirs;
 extern int filesfrom_fd;
 extern int one_file_system;
@@ -49,8 +49,8 @@ extern int preserve_links;
 extern int preserve_hard_links;
 extern int preserve_devices;
 extern int preserve_specials;
-extern int preserve_uid;
-extern int preserve_gid;
+extern int uid_ndx;
+extern int gid_ndx;
 extern int relative_paths;
 extern int implied_dirs;
 extern int file_extra_cnt;
@@ -76,6 +76,8 @@ extern int need_unsorted_flist;
 extern iconv_t ic_send, ic_recv;
 #endif
 
+#define PTR_SIZE (sizeof (struct file_struct *))
+
 int io_error;
 int checksum_len;
 dev_t filesystem_dev; /* used to implement -x */
@@ -305,6 +307,15 @@ static void flist_expand(struct file_list *flist, int extra)
 		out_of_memory("flist_expand");
 }
 
+static void flist_done_allocating(struct file_list *flist)
+{
+	void *ptr = pool_boundary(flist->file_pool, 8*1024);
+	if (flist->pool_boundary == ptr)
+		flist->pool_boundary = NULL; /* list didn't use any pool memory */
+	else
+		flist->pool_boundary = ptr;
+}
+
 int push_pathname(const char *dir, int len)
 {
 	if (dir == pathname)
@@ -402,24 +413,24 @@ static void send_file_entry(int f, struct file_struct *file, int ndx)
 		}
 	} else if (protocol_version < 28)
 		rdev = MAKEDEV(0, 0);
-	if (preserve_uid) {
+	if (uid_ndx) {
 		if ((uid_t)F_OWNER(file) == uid && *lastname)
 			flags |= XMIT_SAME_UID;
 		else {
 			uid = F_OWNER(file);
-			if (preserve_uid && !numeric_ids) {
+			if (uid_ndx && !numeric_ids) {
 				user_name = add_uid(uid);
 				if (inc_recurse && user_name)
 					flags |= XMIT_USER_NAME_FOLLOWS;
 			}
 		}
 	}
-	if (preserve_gid) {
+	if (gid_ndx) {
 		if ((gid_t)F_GROUP(file) == gid && *lastname)
 			flags |= XMIT_SAME_GID;
 		else {
 			gid = F_GROUP(file);
-			if (preserve_gid && !numeric_ids) {
+			if (gid_ndx && !numeric_ids) {
 				group_name = add_gid(gid);
 				if (inc_recurse && group_name)
 					flags |= XMIT_GROUP_NAME_FOLLOWS;
@@ -501,7 +512,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx)
 	}
 	if (!(flags & XMIT_SAME_MODE))
 		write_int(f, to_wire_mode(mode));
-	if (preserve_uid && !(flags & XMIT_SAME_UID)) {
+	if (uid_ndx && !(flags & XMIT_SAME_UID)) {
 		if (protocol_version < 30)
 			write_int(f, uid);
 		else {
@@ -513,7 +524,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx)
 			}
 		}
 	}
-	if (preserve_gid && !(flags & XMIT_SAME_GID)) {
+	if (gid_ndx && !(flags & XMIT_SAME_GID)) {
 		if (protocol_version < 30)
 			write_int(f, gid);
 		else {
@@ -689,9 +700,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 		file_length = F_LENGTH(first);
 		modtime = first->modtime;
 		mode = first->mode;
-		if (preserve_uid)
+		if (uid_ndx)
 			uid = F_OWNER(first);
-		if (preserve_gid)
+		if (gid_ndx)
 			gid = F_GROUP(first);
 		if ((preserve_devices && IS_DEVICE(mode))
 		 || (preserve_specials && IS_SPECIAL(mode))) {
@@ -727,7 +738,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 	if (chmod_modes && !S_ISLNK(mode))
 		mode = tweak_mode(mode, chmod_modes);
 
-	if (preserve_uid && !(xflags & XMIT_SAME_UID)) {
+	if (uid_ndx && !(xflags & XMIT_SAME_UID)) {
 		if (protocol_version < 30)
 			uid = (uid_t)read_int(f);
 		else {
@@ -738,7 +749,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 				uid = match_uid(uid);
 		}
 	}
-	if (preserve_gid && !(xflags & XMIT_SAME_GID)) {
+	if (gid_ndx && !(xflags & XMIT_SAME_GID)) {
 		if (protocol_version < 30)
 			gid = (gid_t)read_int(f);
 		else {
@@ -845,9 +856,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 		OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
 	}
 	file->mode = mode;
-	if (preserve_uid)
+	if (uid_ndx)
 		F_OWNER(file) = uid;
-	if (preserve_gid) {
+	if (gid_ndx) {
 		F_GROUP(file) = gid;
 		file->flags |= gid_flags;
 	}
@@ -991,8 +1002,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	alloc_pool_t *pool;
 	char *bp;
 
-	if (strlcpy(thisname, fname, sizeof thisname)
-	    >= sizeof thisname - pathname_len) {
+	if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) {
 		rprintf(FINFO, "skipping overly long name: %s\n", fname);
 		return NULL;
 	}
@@ -1166,9 +1176,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		OPT_EXTRA(file, 0)->unum = (uint32)(st.st_size >> 32);
 	}
 	file->mode = st.st_mode;
-	if (preserve_uid)
+	if (uid_ndx)
 		F_OWNER(file) = st.st_uid;
-	if (preserve_gid)
+	if (gid_ndx)
 		F_GROUP(file) = st.st_gid;
 
 	if (basename != thisname)
@@ -1197,9 +1207,9 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 			file->modtime = st2.st_mtime;
 			file->len32 = 0;
 			file->mode = st2.st_mode;
-			if (preserve_uid)
+			if (uid_ndx)
 				F_OWNER(file) = st2.st_uid;
-			if (preserve_gid)
+			if (gid_ndx)
 				F_GROUP(file) = st2.st_gid;
 		} else
 			file->mode = save_mode;
@@ -1309,9 +1319,70 @@ static void send_if_directory(int f, struct file_list *flist,
 	}
 }
 
-static int file_compare(struct file_struct **file1, struct file_struct **file2)
+static int file_compare(const void *file1, const void *file2)
 {
-	return f_name_cmp(*file1, *file2);
+	return f_name_cmp(*(struct file_struct **)file1,
+			  *(struct file_struct **)file2);
+}
+
+/* The guts of a merge-sort algorithm.  This was derived from the glibc
+ * version, but I (Wayne) changed the merge code to do less copying and
+ * to require only half the amount of temporary memory. */
+static void fsort_tmp(struct file_struct **fp, size_t num,
+		      struct file_struct **tmp)
+{
+	struct file_struct **f1, **f2, **t;
+	size_t n1, n2;
+
+	n1 = num / 2;
+	n2 = num - n1;
+	f1 = fp;
+	f2 = fp + n1;
+
+	if (n1 > 1)
+		fsort_tmp(f1, n1, tmp);
+	if (n2 > 1)
+		fsort_tmp(f2, n2, tmp);
+
+	while (f_name_cmp(*f1, *f2) <= 0) {
+		if (!--n1)
+			return;
+		f1++;
+	}
+
+	t = tmp;
+	memcpy(t, f1, n1 * PTR_SIZE);
+
+	*f1++ = *f2++, n2--;
+
+	while (n1 > 0 && n2 > 0) {
+		if (f_name_cmp(*t, *f2) <= 0)
+			*f1++ = *t++, n1--;
+		else
+			*f1++ = *f2++, n2--;
+	}
+
+	if (n1 > 0)
+		memcpy(f1, t, n1 * PTR_SIZE);
+}
+
+/* This file-struct sorting routine makes sure that any identical names in
+ * the file list stay in the same order as they were in the original list.
+ * This is particularly vital in inc_recurse mode where we expect a sort
+ * on the flist to match the exact order of a sort on the dir_flist. */
+static void fsort(struct file_struct **fp, size_t num)
+{
+	if (num <= 1)
+		return;
+
+	if (use_qsort)
+		qsort(fp, num, PTR_SIZE, file_compare);
+	else {
+		struct file_struct **tmp = new_array(struct file_struct *,
+						     (num+1) / 2);
+		fsort_tmp(fp, num, tmp);
+		free(tmp);
+	}
 }
 
 /* We take an entire set of sibling dirs from the sorted flist and link them
@@ -1455,55 +1526,34 @@ void send_extra_file_list(int f, int at_least)
 		future_cnt = 0;
 	while (future_cnt < at_least) {
 		struct file_struct *file = dir_flist->sorted[send_dir_ndx];
-		int dstart = dir_count;
+		int dir_ndx, dstart = dir_count;
 		int32 *dp;
 
 		flist = flist_new(0, "send_extra_file_list");
 		start_write = stats.total_written;
 
-		/* If this is the first of a set of duplicate dirs, we must
-		 * send all the dirs together in a single file-list.  We must
-		 * also send the index of the last dir in the header. */
-		if (file->flags & FLAG_DUPLICATE) {
-			int dir_ndx, end_ndx = send_dir_ndx;
-			struct file_struct *fp = file;
-
-			while (1) {
-				dp = F_DIRNODE_P(fp);
-				end_ndx = DIR_NEXT_SIBLING(dp);
-				fp = dir_flist->sorted[end_ndx];
-				if (!(fp->flags & FLAG_DUPLICATE))
-					break;
-			}
-
 #ifdef ICONV_OPTION
-			if (ic_ndx)
-				dir_ndx = F_NDX(fp);
-			else
+		if (ic_ndx)
+			dir_ndx = F_NDX(file);
+		else
 #endif
-				dir_ndx = end_ndx;
-			write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
+			dir_ndx = send_dir_ndx;
+		write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
 
-			while (1) {
-				send1extra(f, file, flist);
-				if (send_dir_ndx == end_ndx)
-					break;
-				dp = F_DIRNODE_P(file);
-				send_dir_ndx = DIR_NEXT_SIBLING(dp);
-				file = dir_flist->sorted[send_dir_ndx];
-			}
-		} else {
-			int dir_ndx;
-#ifdef ICONV_OPTION
-			if (ic_ndx)
-				dir_ndx = F_NDX(file);
-			else
-#endif
-				dir_ndx = send_dir_ndx;
-			write_ndx(f, NDX_FLIST_OFFSET - dir_ndx);
+		send1extra(f, file, flist);
+		dp = F_DIRNODE_P(file);
 
+		/* If there are any duplicate directory names that follow, we
+		 * send all the dirs together in one file-list.  The dir_flist
+		 * tree links all the child subdirs onto the last dup dir. */
+		while ((dir_ndx = DIR_NEXT_SIBLING(dp)) >= 0
+		    && dir_flist->sorted[dir_ndx]->flags & FLAG_DUPLICATE) {
+			send_dir_ndx = dir_ndx;
+			file = dir_flist->sorted[dir_ndx];
 			send1extra(f, file, flist);
+			dp = F_DIRNODE_P(file);
 		}
+
 		write_byte(f, 0);
 
 #ifdef ICONV_OPTION
@@ -1519,6 +1569,7 @@ void send_extra_file_list(int f, int at_least)
 		clean_flist(flist, 0);
 
 		add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart);
+		flist_done_allocating(flist);
 
 		file_total += flist->count;
 		future_cnt += flist->count;
@@ -1527,7 +1578,6 @@ void send_extra_file_list(int f, int at_least)
 		if (verbose > 3)
 			output_flist(flist);
 
-		dp = F_DIRNODE_P(file);
 		if (DIR_FIRST_CHILD(dp) >= 0) {
 			send_dir_ndx = DIR_FIRST_CHILD(dp);
 			send_dir_depth++;
@@ -1853,6 +1903,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
 	if (inc_recurse) {
 		add_dirs_to_tree(-1, flist, dir_count);
+		flist_done_allocating(flist);
 		if (send_dir_ndx < 0) {
 			write_ndx(f, NDX_FLIST_EOF);
 			flist_eof = 1;
@@ -1947,21 +1998,21 @@ struct file_list *recv_file_list(int f)
 						dir_flist->count);
 			memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
 			       (dir_flist->count - dstart) * sizeof (struct file_struct*));
-			qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
-			      sizeof (struct file_struct*), (int (*)())file_compare);
+			fsort(dir_flist->sorted + dstart, dir_flist->count - dstart);
 		}
 	} else
 #endif
 	{
 		flist->sorted = flist->files;
-		if (inc_recurse) {
+		if (inc_recurse && dir_flist->count > dstart) {
 			dir_flist->sorted = dir_flist->files;
-			qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
-			      sizeof (struct file_struct*), (int (*)())file_compare);
+			fsort(dir_flist->sorted + dstart, dir_flist->count - dstart);
 		}
 	}
 
-	if (!inc_recurse && f >= 0)
+	if (inc_recurse)
+		flist_done_allocating(flist);
+	else if (f >= 0)
 		recv_id_list(f, flist);
 
 	clean_flist(flist, relative_paths);
@@ -2011,6 +2062,10 @@ void recv_additional_file_list(int f)
 				NDX_FLIST_OFFSET - dir_flist->count + 1);
 			exit_cleanup(RERR_PROTOCOL);
 		}
+		if (verbose > 3) {
+			rprintf(FINFO, "[%s] receiving flist for dir %d\n",
+				who_am_i(), ndx);
+		}
 		flist = recv_file_list(f);
 		flist->parent_ndx = ndx;
 	}
@@ -2093,34 +2148,42 @@ struct file_list *flist_new(int flags, char *msg)
 
 	memset(flist, 0, sizeof flist[0]);
 
-	if (!(flags & FLIST_TEMP)) {
-		if (first_flist) {
-			flist->ndx_start = first_flist->prev->ndx_start
-					 + first_flist->prev->count;
-		}
+	if (flags & FLIST_TEMP) {
+		if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,
+						out_of_memory, POOL_INTERN)))
+			out_of_memory(msg);
+	} else {
 		/* This is a doubly linked list with prev looping back to
 		 * the end of the list, but the last next pointer is NULL. */
-		if (!first_flist)
+		if (!first_flist) {
+			flist->file_pool = pool_create(NORMAL_EXTENT, 0,
+						out_of_memory, POOL_INTERN);
+			if (!flist->file_pool)
+				out_of_memory(msg);
+
 			first_flist = cur_flist = flist->prev = flist;
-		else {
+		} else {
+			flist->file_pool = first_flist->file_pool;
+
+			flist->ndx_start = first_flist->prev->ndx_start
+					 + first_flist->prev->count;
+
 			flist->prev = first_flist->prev;
 			flist->prev->next = first_flist->prev = flist;
 		}
+		flist->pool_boundary = pool_boundary(flist->file_pool, 0);
 		flist_cnt++;
 	}
 
-	if (!(flist->file_pool = pool_create(FILE_EXTENT, 0, out_of_memory, POOL_INTERN)))
-		out_of_memory(msg);
-
 	return flist;
 }
 
 /* Free up all elements in a flist. */
 void flist_free(struct file_list *flist)
 {
-	if (!flist->prev)
-		; /* Was FLIST_TEMP dir-list. */
-	else if (flist == flist->prev) {
+	if (!flist->prev) {
+		/* Was FLIST_TEMP dir-list. */
+	} else if (flist == flist->prev) {
 		first_flist = cur_flist = NULL;
 		file_total = 0;
 		flist_cnt = 0;
@@ -2139,7 +2202,11 @@ void flist_free(struct file_list *flist)
 		flist_cnt--;
 	}
 
-	pool_destroy(flist->file_pool);
+	if (!flist->prev || !flist_cnt)
+		pool_destroy(flist->file_pool);
+	else
+		pool_free_old(flist->file_pool, flist->pool_boundary);
+
 	if (flist->sorted && flist->sorted != flist->files)
 		free(flist->sorted);
 	free(flist->files);
@@ -2160,8 +2227,7 @@ static void clean_flist(struct file_list *flist, int strip_root)
 		return;
 	}
 
-	qsort(flist->sorted, flist->count,
-	    sizeof flist->sorted[0], (int (*)())file_compare);
+	fsort(flist->sorted, flist->count);
 
 	if (!am_sender || inc_recurse) {
 		for (i = prev_i = 0; i < flist->count; i++) {
@@ -2195,20 +2261,18 @@ static void clean_flist(struct file_list *flist, int strip_root)
 		} else
 			j = -1;
 		if (j >= 0) {
-			struct file_struct *fp = flist->sorted[j];
 			int keep, drop;
 			/* If one is a dir and the other is not, we want to
 			 * keep the dir because it might have contents in the
 			 * list. */
-			if (S_ISDIR(file->mode) != S_ISDIR(fp->mode)) {
-				if (S_ISDIR(file->mode))
+			if (S_ISDIR(file->mode)) {
+				struct file_struct *fp = flist->sorted[j];
+				if (!S_ISDIR(fp->mode))
 					keep = i, drop = j;
 				else
 					keep = j, drop = i;
-			} else if (protocol_version < 27)
+			} else
 				keep = j, drop = i;
-			else
-				keep = i, drop = j;
 
 			if (am_sender)
 				flist->sorted[drop]->flags |= FLAG_DUPLICATE;
@@ -2334,12 +2398,12 @@ static void output_flist(struct file_list *flist)
 		who, flist->ndx_start, flist->count, flist->low, flist->high);
 	for (i = 0; i < flist->count; i++) {
 		file = flist->sorted[i];
-		if ((am_root || am_sender) && preserve_uid) {
+		if ((am_root || am_sender) && uid_ndx) {
 			snprintf(uidbuf, sizeof uidbuf, " uid=%u",
 				 F_OWNER(file));
 		} else
 			*uidbuf = '\0';
-		if (preserve_gid) {
+		if (gid_ndx) {
 			static char parens[] = "(\0)\0\0\0";
 			char *pp = parens + (file->flags & FLAG_SKIP_GROUP ? 0 : 3);
 			snprintf(gidbuf, sizeof gidbuf, " gid=%s%u%s",