One more tweak.

[rsync/rsync-patches.git] / filter.diff

After applying this patch and running configure, you MUST run this
command before "make":

    make proto

This patch adds the ability to merge rules into your excludes/includes
using a ". FILE" idiom.  If you specify a name without slashes, that
filename will be looked for in every subdirectory that rsync visits,
and its rules will affect the current directory and its subdirectories.

For example:

  rsync -av --exclude='. .excl' from/ to

The above will look for a file named ".excl" in every directory of the
hierarchy that rsync visits, and it will exclude (by default) names
based on the rules found therein.  If one of the .excl files contains
this:

  + *.c
  . .excl2
  . ./.excl3
  *.o

Then the file ".excl2" will also be read in the current dir, and all
subdirs of the current dir.  The file ".excl3" would just be read in
for the current dir because its name contained a slash.

..wayne..

--- exclude.c	14 May 2004 21:23:41 -0000	1.76
+++ exclude.c	15 May 2004 02:45:44 -0000
@@ -30,32 +30,58 @@ extern int verbose;
 extern int eol_nulls;
 extern int list_only;
 extern int recurse;
+extern int io_error;
+extern int sanitize_paths;
 
 extern char curr_dir[];
 
-struct exclude_list_struct exclude_list = { 0, 0, "" };
-struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " };
-struct exclude_list_struct server_exclude_list = { 0, 0, "server " };
+struct exclude_list_struct exclude_list = { 0, 0, 0, "" };
+struct exclude_list_struct server_exclude_list = { 0, 0, 0, "server " };
 char *exclude_path_prefix = NULL;
 
+struct exclude_list_root {
+    struct exclude_list_struct *head;
+    int cnt;
+} local_lists;
+
+static char dirbuf[MAXPATHLEN];
+static unsigned int dirbuf_offset = 0;
+
 /** Build an exclude structure given a exclude pattern */
-static void make_exclude(struct exclude_list_struct *listp, const char *pattern,
-			 int pat_len, int include)
+static void make_exclude(struct exclude_list_struct *listp, const char *pat,
+			 unsigned int pat_len, int mflags)
 {
 	struct exclude_struct *ret;
 	const char *cp;
-	int ex_len;
+	unsigned int ex_len;
+
+	if (mflags & MATCHFLG_MERGE_FILE) {
+		struct exclude_struct *ex;
+		/* If the local include file was already mentioned, don't
+		 * add it again. */
+		for (ex = listp->head; ex; ex = ex->next) {
+			if ((ex->match_flags & MATCHFLG_MERGE_FILE)
+			    && strlen(ex->pattern) == pat_len
+			    && strncmp(ex->pattern, pat, pat_len) == 0)
+				return;
+		}
+		if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/'))
+		    && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) {
+			mflags |= MATCHFLG_CVSIGNORE;
+			mflags &= ~MATCHFLG_INCLUDE;
+		} else
+			mflags &= ~MATCHFLG_CVSIGNORE;
+	}
 
 	ret = new(struct exclude_struct);
 	if (!ret)
 		out_of_memory("make_exclude");
 
 	memset(ret, 0, sizeof ret[0]);
-	ret->include = include;
 
 	if (exclude_path_prefix)
-		ret->match_flags |= MATCHFLG_ABS_PATH;
-	if (exclude_path_prefix && *pattern == '/')
+		mflags |= MATCHFLG_ABS_PATH;
+	if (exclude_path_prefix && *pat == '/')
 		ex_len = strlen(exclude_path_prefix);
 	else
 		ex_len = 0;
@@ -64,33 +90,52 @@ static void make_exclude(struct exclude_
 		out_of_memory("make_exclude");
 	if (ex_len)
 		memcpy(ret->pattern, exclude_path_prefix, ex_len);
-	strlcpy(ret->pattern + ex_len, pattern, pat_len + 1);
+	strlcpy(ret->pattern + ex_len, pat, pat_len + 1);
 	pat_len += ex_len;
 
 	if (strpbrk(ret->pattern, "*[?")) {
-		ret->match_flags |= MATCHFLG_WILD;
+		mflags |= MATCHFLG_WILD;
 		if ((cp = strstr(ret->pattern, "**")) != NULL) {
-			ret->match_flags |= MATCHFLG_WILD2;
+			mflags |= MATCHFLG_WILD2;
 			/* If the pattern starts with **, note that. */
 			if (cp == ret->pattern)
-				ret->match_flags |= MATCHFLG_WILD2_PREFIX;
+				mflags |= MATCHFLG_WILD2_PREFIX;
 		}
 	}
 
 	if (pat_len > 1 && ret->pattern[pat_len-1] == '/') {
 		ret->pattern[pat_len-1] = 0;
-		ret->directory = 1;
+		mflags |= MATCHFLG_DIRECTORY;
 	}
 
 	for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++)
 		ret->slash_cnt++;
 
-	if (!listp->tail)
+	if (!listp->tail) {
+		ret->next = listp->head;
 		listp->head = listp->tail = ret;
-	else {
+	} else {
+		ret->next = listp->tail->next;
 		listp->tail->next = ret;
 		listp->tail = ret;
 	}
+
+	if (mflags & MATCHFLG_MERGE_FILE) {
+		struct exclude_list_struct *lp;
+		int ndx = local_lists.cnt++;
+		local_lists.head = realloc_array(local_lists.head,
+		    struct exclude_list_struct, local_lists.cnt);
+		if (!local_lists.head)
+			out_of_memory("make_exclude");
+		lp = &local_lists.head[ndx];
+		lp->head = lp->tail = NULL;
+		if (asprintf(&lp->debug_type, "per-dir %s ", ret->pattern) < 0)
+			out_of_memory("make_exclude");
+		lp->parent = ret;
+		ret->slash_cnt = ndx;
+	}
+
+	ret->match_flags = mflags;
 }
 
 static void free_exclude(struct exclude_struct *ex)
@@ -99,18 +144,90 @@ static void free_exclude(struct exclude_
 	free(ex);
 }
 
-void free_exclude_list(struct exclude_list_struct *listp)
+static void free_exclude_list(struct exclude_list_struct *listp)
 {
-	struct exclude_struct *ent, *next;
-
-	for (ent = listp->head; ent; ent = next) {
-		next = ent->next;
-		free_exclude(ent);
+	if (listp->tail) {
+		struct exclude_struct *ent, *next;
+		/* Truncate any extra parent items from local list. */
+		listp->tail->next = NULL;
+		for (ent = listp->head; ent; ent = next) {
+			next = ent->next;
+			free_exclude(ent);
+		}
 	}
 
 	listp->head = listp->tail = NULL;
 }
 
+void *push_local_excludes(char *fname, unsigned int offset)
+{
+	int i;
+	struct exclude_list_root *push = new_array(struct exclude_list_root, 1);
+
+	if (!push)
+		out_of_memory("push_local_excludes");
+
+	push->cnt = local_lists.cnt;
+	push->head = new_array(struct exclude_list_struct, local_lists.cnt);
+	if (!push->head)
+		out_of_memory("push_local_excludes");
+
+	memcpy(push->head, local_lists.head,
+	    sizeof (struct exclude_list_struct) * local_lists.cnt);
+
+	/* Make it easy to construct the full path for a merge that has
+	 * a relative path by saving it off. */
+	memcpy(dirbuf, fname, offset);
+	dirbuf_offset = offset;
+
+	for (i = 0; i < local_lists.cnt; i++) {
+		struct exclude_list_struct *listp = &local_lists.head[i];
+		char *file = listp->parent->pattern;
+		int flags;
+
+		if (verbose > 2) {
+			rprintf(FINFO, "[%s] pushing %sexclude list\n",
+				who_am_i(), listp->debug_type);
+		}
+		if (listp->parent->match_flags & MATCHFLG_CVSIGNORE) {
+			flags = XFLG_WORD_SPLIT | XFLG_WORDS_ONLY;
+			listp->head = NULL; /* Subdirs don't get our rules. */
+		} else {
+			flags = listp->parent->match_flags & MATCHFLG_INCLUDE
+			    ? XFLG_DEF_INCLUDE : 0;
+		}
+		listp->tail = NULL;
+		if (strlcpy(fname +  offset, file, MAXPATHLEN - offset)
+		    < MAXPATHLEN - offset)
+			add_exclude_file(listp, fname, flags);
+		else {
+			io_error |= IOERR_GENERAL;
+			rprintf(FINFO,
+			    "cannot add local excludes in long-named directory %s\n",
+			    full_fname(fname));
+		}
+	}
+
+	return (void*)push;
+}
+
+void pop_local_excludes(void *mem)
+{
+	int i;
+
+	for (i = 0; i < local_lists.cnt; i++) {
+		struct exclude_list_struct *listp = &local_lists.head[i];
+		if (verbose > 2) {
+			rprintf(FINFO, "[%s] popping %sexclude list\n",
+				who_am_i(), listp->debug_type);
+		}
+		free_exclude_list(listp);
+	}
+	free(local_lists.head);
+	local_lists = *(struct exclude_list_root*)mem;
+	free(mem);
+}
+
 static int check_one_exclude(char *name, struct exclude_struct *ex,
                              int name_is_dir)
 {
@@ -134,7 +251,8 @@ static int check_one_exclude(char *name,
 
 	if (!name[0]) return 0;
 
-	if (ex->directory && !name_is_dir) return 0;
+	if ((ex->match_flags & MATCHFLG_DIRECTORY) && !name_is_dir)
+		return 0;
 
 	if (*pattern == '/') {
 		match_start = 1;
@@ -201,9 +319,11 @@ static void report_exclude_result(char c
 
 	if (verbose >= 2) {
 		rprintf(FINFO, "[%s] %scluding %s %s because of %spattern %s%s\n",
-			who_am_i(), ent->include ? "in" : "ex",
+			who_am_i(),
+			ent->match_flags & MATCHFLG_INCLUDE ? "in" : "ex",
 			name_is_dir ? "directory" : "file", name, type,
-			ent->pattern, ent->directory ? "/" : "");
+			ent->pattern,
+			ent->match_flags & MATCHFLG_DIRECTORY ? "/" : "");
 	}
 }
 
@@ -217,10 +337,18 @@ int check_exclude(struct exclude_list_st
 	struct exclude_struct *ent;
 
 	for (ent = listp->head; ent; ent = ent->next) {
+		if (ent->match_flags & MATCHFLG_MERGE_FILE) {
+			struct exclude_list_struct *lp
+			    = &local_lists.head[ent->slash_cnt];
+			int rc = check_exclude(lp, name, name_is_dir);
+			if (rc)
+				return rc;
+			continue;
+		}
 		if (check_one_exclude(name, ent, name_is_dir)) {
 			report_exclude_result(name, ent, name_is_dir,
 					      listp->debug_type);
-			return ent->include ? 1 : -1;
+			return (ent->match_flags & MATCHFLG_INCLUDE) ? 1 : -1;
 		}
 	}
 
@@ -236,11 +364,11 @@ int check_exclude(struct exclude_list_st
  * *incl_ptr value will be 1 for an include, 0 for an exclude, and -1 for
  * the list-clearing "!" token.
  */
-static const char *get_exclude_tok(const char *p, int *len_ptr, int *incl_ptr,
+static const char *get_exclude_tok(const char *p, int *len_ptr, int *flag_ptr,
 				   int xflags)
 {
 	const unsigned char *s = (const unsigned char *)p;
-	int len;
+	int len, mflags = 0;
 
 	if (xflags & XFLG_WORD_SPLIT) {
 		/* Skip over any initial whitespace. */
@@ -250,13 +378,19 @@ static const char *get_exclude_tok(const
 		p = (const char *)s;
 	}
 
-	/* Is this a '+' or '-' followed by a space (not whitespace)? */
+	/* Is this a +/-/. followed by a space (not whitespace)? */
 	if (!(xflags & XFLG_WORDS_ONLY)
-	    && (*s == '-' || *s == '+') && s[1] == ' ') {
-		*incl_ptr = *s == '+';
+	    && (*s == '-' || *s == '+' || *s == '.') && s[1] == ' ') {
+		if (*s == '+')
+			mflags |= MATCHFLG_INCLUDE;
+		else if (*s == '.') {
+			mflags |= MATCHFLG_MERGE_FILE;
+			if (xflags & XFLG_DEF_INCLUDE)
+				mflags |= MATCHFLG_INCLUDE;
+		}
 		s += 2;
-	} else
-		*incl_ptr = xflags & XFLG_DEF_INCLUDE;
+	} else if (xflags & XFLG_DEF_INCLUDE)
+		mflags |= MATCHFLG_INCLUDE;
 
 	if (xflags & XFLG_WORD_SPLIT) {
 		const unsigned char *cp = s;
@@ -268,9 +402,10 @@ static const char *get_exclude_tok(const
 		len = strlen(s);
 
 	if (*p == '!' && len == 1 && !(xflags & XFLG_WORDS_ONLY))
-		*incl_ptr = -1;
+		mflags |= MATCHFLG_CLEAR_LIST;
 
 	*len_ptr = len;
+	*flag_ptr = mflags;
 	return (const char *)s;
 }
 
@@ -278,7 +413,7 @@ static const char *get_exclude_tok(const
 void add_exclude(struct exclude_list_struct *listp, const char *pattern,
 		 int xflags)
 {
-	int pat_len, incl;
+	int pat_len, mflags;
 	const char *cp;
 
 	if (!pattern)
@@ -287,27 +422,49 @@ void add_exclude(struct exclude_list_str
 	cp = pattern;
 	pat_len = 0;
 	while (1) {
-		cp = get_exclude_tok(cp + pat_len, &pat_len, &incl, xflags);
+		cp = get_exclude_tok(cp + pat_len, &pat_len, &mflags, xflags);
 		if (!pat_len)
 			break;
-		/* If we got the special "!" token, clear the list. */
-		if (incl < 0) {
+		if (mflags & MATCHFLG_CLEAR_LIST) {
 			if (verbose > 2) {
 				rprintf(FINFO,
 					"[%s] clearing %sexclude list\n",
 					who_am_i(), listp->debug_type);
 			}
 			free_exclude_list(listp);
-		} else {
-			make_exclude(listp, cp, pat_len, incl);
-
-			if (verbose > 2) {
-				rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s)\n",
-					who_am_i(), pat_len, cp,
-					listp->debug_type,
-					incl ? "include" : "exclude");
+			continue;
+		}
+		if (mflags & MATCHFLG_MERGE_FILE) {
+			char name[MAXPATHLEN];
+			if ((unsigned) pat_len >= sizeof name)
+				continue; /* XXX complain? */
+			strlcpy(name, cp, pat_len+1);
+			if (strchr(name, '/') != NULL) {
+				if (sanitize_paths)
+					sanitize_path(name, curr_dir);
+				if (*name == '/')
+					cp = name;
+				else {
+					if (strlcpy(dirbuf + dirbuf_offset,
+					    name, MAXPATHLEN - dirbuf_offset)
+					    >= MAXPATHLEN - dirbuf_offset)
+						continue; /* XXX complain? */
+					cp = dirbuf;
+				}
+				add_exclude_file(listp, cp,
+				    xflags | XFLG_FATAL_ERRORS);
+				continue;
 			}
 		}
+
+		make_exclude(listp, cp, pat_len, mflags);
+
+		if (verbose > 2) {
+			rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n",
+				who_am_i(), pat_len, cp, listp->debug_type,
+				mflags & MATCHFLG_MERGE_FILE ? "FILE " : "",
+				mflags & MATCHFLG_INCLUDE ? "in" : "ex");
+		}
 	}
 }
 
@@ -383,15 +540,19 @@ void send_exclude_list(int f)
 		l = strlcpy(p, ent->pattern, sizeof p);
 		if (l == 0 || l >= MAXPATHLEN)
 			continue;
-		if (ent->directory) {
+		if (ent->match_flags & MATCHFLG_DIRECTORY) {
 			p[l++] = '/';
 			p[l] = '\0';
 		}
 
-		if (ent->include) {
+		if (ent->match_flags & MATCHFLG_INCLUDE) {
 			write_int(f, l + 2);
 			write_buf(f, "+ ", 2);
-		} else if ((*p == '-' || *p == '+') && p[1] == ' ') {
+		} else if (ent->match_flags & MATCHFLG_MERGE_FILE) {
+			write_int(f, l + 2);
+			write_buf(f, ". ", 2);
+		} else if ((*p == '-' || *p == '+' || *p == '.')
+		    && p[1] == ' ') {
 			write_int(f, l + 2);
 			write_buf(f, "- ", 2);
 		} else
@@ -432,6 +593,7 @@ void add_cvs_excludes(void)
 	char fname[MAXPATHLEN];
 	char *p;
 
+	add_exclude(&exclude_list, ". .cvsignore", 0);
 	add_exclude(&exclude_list, default_cvsignore,
 		    XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
 
--- flist.c	14 May 2004 21:23:41 -0000	1.222
+++ flist.c	15 May 2004 02:45:44 -0000
@@ -39,8 +39,6 @@ extern int module_id;
 extern int ignore_errors;
 extern int numeric_ids;
 
-extern int cvs_exclude;
-
 extern int recurse;
 extern char curr_dir[MAXPATHLEN];
 extern char *files_from;
@@ -65,7 +63,6 @@ extern int write_batch;
 
 extern struct exclude_list_struct exclude_list;
 extern struct exclude_list_struct server_exclude_list;
-extern struct exclude_list_struct local_exclude_list;
 
 int io_error;
 
@@ -210,8 +207,6 @@ int link_stat(const char *path, STRUCT_S
  */
 static int check_exclude_file(char *fname, int is_dir, int exclude_level)
 {
-	int rc;
-
 #if 0 /* This currently never happens, so avoid a useless compare. */
 	if (exclude_level == NO_EXCLUDES)
 		return 0;
@@ -233,10 +228,7 @@ static int check_exclude_file(char *fnam
 	if (exclude_level != ALL_EXCLUDES)
 		return 0;
 	if (exclude_list.head
-	    && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0)
-		return rc < 0;
-	if (local_exclude_list.head
-	    && check_exclude(&local_exclude_list, fname, is_dir) < 0)
+	    && check_exclude(&exclude_list, fname, is_dir) < 0)
 		return 1;
 	return 0;
 }
@@ -946,15 +938,7 @@ void send_file_name(int f, struct file_l
 
 	if (recursive && S_ISDIR(file->mode)
 	    && !(file->flags & FLAG_MOUNT_POINT)) {
-		struct exclude_list_struct last_list = local_exclude_list;
-		local_exclude_list.head = local_exclude_list.tail = NULL;
 		send_directory(f, flist, f_name_to(file, fbuf));
-		if (verbose > 2) {
-			rprintf(FINFO, "[%s] popping %sexclude list\n",
-				who_am_i(), local_exclude_list.debug_type);
-		}
-		free_exclude_list(&local_exclude_list);
-		local_exclude_list = last_list;
 	}
 }
 
@@ -965,6 +949,7 @@ static void send_directory(int f, struct
 	struct dirent *di;
 	char fname[MAXPATHLEN];
 	unsigned int offset;
+	void *save_excludes;
 	char *p;
 
 	d = opendir(dir);
@@ -989,18 +974,7 @@ static void send_directory(int f, struct
 		offset++;
 	}
 
-	if (cvs_exclude) {
-		if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset)
-		    < MAXPATHLEN - offset) {
-			add_exclude_file(&local_exclude_list, fname,
-					 XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
-		} else {
-			io_error |= IOERR_GENERAL;
-			rprintf(FINFO,
-				"cannot cvs-exclude in long-named directory %s\n",
-				full_fname(fname));
-		}
-	}
+	save_excludes = push_local_excludes(fname, offset);
 
 	for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
 		char *dname = d_name(di);
@@ -1021,6 +995,8 @@ static void send_directory(int f, struct
 		rprintf(FERROR, "readdir(%s): (%d) %s\n",
 			dir, errno, strerror(errno));
 	}
+
+	pop_local_excludes(save_excludes);
 
 	closedir(d);
 }
--- rsync.h	13 May 2004 18:51:22 -0000	1.203
+++ rsync.h	15 May 2004 02:45:45 -0000
@@ -493,18 +493,21 @@ struct map_struct {
 #define MATCHFLG_WILD2		(1<<1) /* pattern has '**' */
 #define MATCHFLG_WILD2_PREFIX	(1<<2) /* pattern starts with '**' */
 #define MATCHFLG_ABS_PATH	(1<<3) /* path-match on absolute path */
+#define MATCHFLG_INCLUDE	(1<<4) /* this is an include, not an exclude */
+#define MATCHFLG_CLEAR_LIST 	(1<<5) /* this item is the "!" token */
+#define MATCHFLG_DIRECTORY	(1<<6) /* this matches only directories */
+#define MATCHFLG_MERGE_FILE	(1<<7) /* specifies a file to merge */
+#define MATCHFLG_CVSIGNORE	(1<<8) /* parse this as a .cvsignore file */
 struct exclude_struct {
 	struct exclude_struct *next;
 	char *pattern;
 	int match_flags;
-	int include;
-	int directory;
 	int slash_cnt;
 };
 
 struct exclude_list_struct {
-	struct exclude_struct *head;
-	struct exclude_struct *tail;
+	struct exclude_struct *head, *tail;
+	struct exclude_struct *parent;
 	char *debug_type;
 };
 
--- rsync.yo	7 May 2004 00:18:37 -0000	1.169
+++ rsync.yo	15 May 2004 02:45:45 -0000
@@ -1075,6 +1075,72 @@ itemize(
   it would be excluded by the "*")
 )
 
+manpagesection(MERGING EXCLUDE FILES)
+
+You can merge whole files into an exclude file using a rule that starts
+with a ". " (a dot followed by a space) and has a filename in place of the
+pattern.  There are two types of merge rules, single-instance and
+per-directory:
+
+itemize(
+  it() If the filename has no slashes in it, it is a per-directory merge;
+  rsync scans every directory that is traversed and merges the named file's
+  contents (when it exists), putting the contents of each subdirectory's
+  file at the start of this per-directory sub-list (so subdirectories
+  inherit the contents of their parent directories by default, but each
+  subdirectory's rules have precedence over the parent's rules).
+
+  it() If a filename has a slash in it, it is a single-instance merge; the
+  named file's contents will be merged into the current exclude file,
+  replacing the merge rule.  Thus, you should use the name ./foo instead of
+  foo if you don't want to scan for "foo" in all the subdirectories of the
+  current directory.
+)
+
+Note also that you can eliminate all the inherited rules for the current
+per-directory ruleset by putting the list-clearing token (!) in the file.
+This clears only the rules of the current per-directory sub-list (up
+through the token) and only for the current directory and its
+subdirectories.
+
+Here's an example. Specify the file that holds this set of rules via a
+normal --exclude-from option:
+
+verb(
+    . /home/user/.global_excludes
+    - *.gz
+    . .excl
+    + *.[ch]
+    - *.o
+)
+
+This will merge the contents of the /home/user/.global_excludes file at the
+start of the list and also turns the ".excl" filename into a per-directory
+exclude file whose local contents will be merged into the list in place of
+the .excl line.
+
+Additionally, you can affect where the --cvs-exclude (-C) option's
+inclusion of a per-directory .cvsignore file gets placed into your rules by
+adding an explicit a merge rule for ".cvsignore".  For instance, specifying
+this:
+
+verb(
+  rsync -avC --exclude='. .cvsignore' --exclude-from=foo a/ b
+)
+
+will merge all the per-directory .cvsignore rules at the start of your list
+rather than at the end.  This allows their dir-specific rules to supersede
+your rules instead of being subservient to them.  (The global rules taken
+from the $HOME/.cvsignore file and from $CVSIGNORE are not affected by
+this.)
+
+Note also that the parsing of any merge-file named ".cvsignore" is always
+done in a CVS-compatible manner (even if -C wasn't specified) -- i.e. the
+rules are always exclude rules (even when specified by an include option),
+they are split on whitespace, no special prefixes or list-clearing tokens
+are honored, and (for per-directory files) subdirectories don't inherit the
+parent directory's rules.
+
 manpagesection(BATCH MODE)
 
 bf(Note:) Batch mode should be considered experimental in this version