From 524989ae1cc10ed0cba23bd2337b73853a6f5679 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sat, 24 Apr 2004 18:49:08 +0000 Subject: [PATCH] - Refer to this rule as a "merge" rather than an "insert" to avoid confusion with "include". - Made it possible to add new per-directory merge files from inside a per-directory merge file. - Fixed a couple bugs. - Added documentation. --- filter.diff | 289 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 194 insertions(+), 95 deletions(-) diff --git a/filter.diff b/filter.diff index 8c3de6e..d5b9e3c 100644 --- a/filter.diff +++ b/filter.diff @@ -1,40 +1,31 @@ -This patch adds the ability to use ". INSERT" files in excludes and -includes. If you specify a name without slashes, that filename will be -looked for in every directory and its rules will effect that directory -and its subdirectories. Insert rules found inside a per-directory -include file are always just read in (they don't create any new per-dir -include/exclude files). +This patch adds the ability to merge rules into your excludes/includes +using a ". FILE" idiom. If you specify a name without slashes, that +filename will be looked for in every subdirectory that rsync visits, +and its rules will affect the current directory and its subdirectories. For example: rsync -av --exclude='. .excl' from/ to The above will look for a file named ".excl" in every directory of the -sender and will exclude (by default) files based on the rules found -therein. If a file contains this: +hierarchy that rsync visits, and it will exclude (by default) names +based on the rules found therein. If one of the .excl files contains +this: + *.c - . another.file - - *.o + . .excl2 + . ./.excl3 + *.o -Then the file "another.file" will also be read (from that one dir) and -its rules inserted in between the surrounding lines. - -Additionally, you can affect where the -C option's inclusion of the -.cvsignore file gets inserted into your rules by mentioning it as an -insertion. For instance, specifying this: - - rsync -avC --include=foo --exclude='. .cvsignore' --include='*.c' a/ b - -This will insert all the .cvsignore rules in the middle of your rules -rather than at the end. This allows their dir-specific rules to -supersede your general rules instead of being subservient to them. +Then the file ".excl2" will also be read in the current dir, and all +subdirs of the current dir. The file ".excl3" would just be read in +for the current dir because its name contained a slash. ..wayne.. ---- exclude.c 22 Apr 2004 22:17:15 -0000 1.71 -+++ exclude.c 22 Apr 2004 23:45:51 -0000 -@@ -30,31 +30,60 @@ extern int verbose; +--- exclude.c 24 Apr 2004 08:00:39 -0000 1.73 ++++ exclude.c 24 Apr 2004 18:25:55 -0000 +@@ -30,32 +30,65 @@ extern int verbose; extern int eol_nulls; extern int list_only; extern int recurse; @@ -44,14 +35,17 @@ supersede your general rules instead of being subservient to them. extern char curr_dir[]; -struct exclude_list_struct exclude_list = { 0, 0, "" }; --struct exclude_list_struct local_exclude_list = { 0, 0, "local-cvsignore " }; +-struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " }; -struct exclude_list_struct server_exclude_list = { 0, 0, "server " }; +struct exclude_list_struct exclude_list = { 0, 0, 0, 0, "" }; +struct exclude_list_struct server_exclude_list = { 0, 0, 0, 0, "server " }; char *exclude_path_prefix = NULL; -+static struct exclude_list_struct *local_exclude_lists; -+static int local_exclude_list_cnt; ++struct exclude_list_root { ++ struct exclude_list_struct *head; ++ int cnt; ++} local_lists; ++ +static char dirbuf[MAXPATHLEN]; +static unsigned int dirbuf_offset = 0; + @@ -63,8 +57,9 @@ supersede your general rules instead of being subservient to them. +} + /** Build an exclude structure given a exclude pattern */ - static void make_exclude(struct exclude_list_struct *listp, const char *pattern, +-static void make_exclude(struct exclude_list_struct *listp, const char *pattern, - int pat_len, int include) ++static void make_exclude(struct exclude_list_struct *listp, const char *pat, + unsigned int pat_len, int mflags) { struct exclude_struct *ret; @@ -72,17 +67,18 @@ supersede your general rules instead of being subservient to them. - int ex_len; + unsigned int ex_len; + -+ if (mflags & MATCHFLG_INSERT_FILE) { ++ if (mflags & MATCHFLG_MERGE_FILE) { + struct exclude_struct *ex; + /* If the local include file was already mentioned, don't -+ * insert it again. */ ++ * add it again. */ + for (ex = listp->head; ex; ex = ex->next) { -+ if ((ex->match_flags & MATCHFLG_INSERT_FILE) ++ if ((ex->match_flags & MATCHFLG_MERGE_FILE) + && strlen(ex->pattern) == pat_len -+ && strncmp(ex->pattern, pattern, pat_len) == 0) ++ && strncmp(ex->pattern, pat, pat_len) == 0) + return; + } -+ if (pat_len == 10 && strncmp(pattern, ".cvsignore", 10) == 0) { ++ if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/')) ++ && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) { + mflags |= MATCHFLG_CVSIGNORE; + mflags &= ~MATCHFLG_INCLUDE; + } else @@ -98,11 +94,18 @@ supersede your general rules instead of being subservient to them. if (exclude_path_prefix) - ret->match_flags |= MATCHFLG_ABS_PATH; +- if (exclude_path_prefix && *pattern == '/') + mflags |= MATCHFLG_ABS_PATH; - if (exclude_path_prefix && *pattern == '/') ++ if (exclude_path_prefix && *pat == '/') ex_len = strlen(exclude_path_prefix); else -@@ -68,29 +97,49 @@ static void make_exclude(struct exclude_ + ex_len = 0; +@@ -64,33 +98,52 @@ static void make_exclude(struct exclude_ + out_of_memory("make_exclude"); + if (ex_len) + memcpy(ret->pattern, exclude_path_prefix, ex_len); +- strlcpy(ret->pattern + ex_len, pattern, pat_len + 1); ++ strlcpy(ret->pattern + ex_len, pat, pat_len + 1); pat_len += ex_len; if (strpbrk(ret->pattern, "*[?")) { @@ -136,17 +139,16 @@ supersede your general rules instead of being subservient to them. listp->tail = ret; } + -+ if (mflags & MATCHFLG_INSERT_FILE) { ++ if (mflags & MATCHFLG_MERGE_FILE) { + struct exclude_list_struct *lp; -+ int ndx = local_exclude_list_cnt++; -+ local_exclude_lists = realloc_array(local_exclude_lists, -+ struct exclude_list_struct, local_exclude_list_cnt); -+ if (!local_exclude_lists) ++ int ndx = local_lists.cnt++; ++ local_lists.head = realloc_array(local_lists.head, ++ struct exclude_list_struct, local_lists.cnt); ++ if (!local_lists.head) + out_of_memory("make_exclude"); -+ lp = &local_exclude_lists[ndx]; ++ lp = &local_lists.head[ndx]; + clear_exclude_list(lp, NULL); -+ if (asprintf(&lp->debug_type, "local %s ", -+ ret->pattern) < 0) ++ if (asprintf(&lp->debug_type, "per-dir %s ", ret->pattern) < 0) + out_of_memory("make_exclude"); + lp->parent = ret; + ret->slash_cnt = ndx; @@ -156,7 +158,7 @@ supersede your general rules instead of being subservient to them. } static void free_exclude(struct exclude_struct *ex) -@@ -99,7 +148,7 @@ static void free_exclude(struct exclude_ +@@ -99,13 +152,15 @@ static void free_exclude(struct exclude_ free(ex); } @@ -165,19 +167,18 @@ supersede your general rules instead of being subservient to them. { struct exclude_struct *ent, *next; -@@ -108,12 +157,72 @@ void free_exclude_list(struct exclude_li - who_am_i(), listp->debug_type); - } - +- if (verbose > 2) { +- rprintf(FINFO, "[%s] clearing %sexclude list\n", +- who_am_i(), listp->debug_type); + if (listp->extra) { + if (listp->tail) + listp->tail->next = NULL; + else + listp->head = NULL; -+ } -+ + } + for (ent = listp->head; ent; ent = next) { - next = ent->next; +@@ -113,7 +168,78 @@ void free_exclude_list(struct exclude_li free_exclude(ent); } @@ -188,34 +189,47 @@ supersede your general rules instead of being subservient to them. +void *push_local_excludes(char *fname, unsigned int offset) +{ + int i; -+ struct exclude_list_struct *mem = new_array(struct exclude_list_struct, -+ local_exclude_list_cnt); -+ if (!mem) ++ struct exclude_list_root *push = new_array(struct exclude_list_root, 1); ++ ++ if (!push) + out_of_memory("push_local_excludes"); + -+ memcpy(mem, local_exclude_lists, -+ sizeof (struct exclude_list_struct) * local_exclude_list_cnt); ++ push->cnt = local_lists.cnt; ++ push->head = new_array(struct exclude_list_struct, local_lists.cnt); ++ if (!push->head) ++ out_of_memory("push_local_excludes"); + ++ memcpy(push->head, local_lists.head, ++ sizeof (struct exclude_list_struct) * local_lists.cnt); ++ ++ /* Make it easy to construct the full path for a merge that has ++ * a relative path by saving it off. */ + memcpy(dirbuf, fname, offset); + dirbuf_offset = offset; + -+ for (i = 0; i < local_exclude_list_cnt; i++) { -+ struct exclude_list_struct *listp = &local_exclude_lists[i]; ++ for (i = 0; i < local_lists.cnt; i++) { ++ struct exclude_list_struct *listp = &local_lists.head[i]; + struct exclude_struct *extra; + char *file = listp->parent->pattern; + int flags; ++ ++ if (verbose > 2) { ++ rprintf(FINFO, "[%s] pushing %sexclude list\n", ++ who_am_i(), listp->debug_type); ++ } + if (listp->parent->match_flags & MATCHFLG_CVSIGNORE) { + flags = XFLG_WORD_SPLIT | XFLG_NO_PREFIXES; + extra = NULL; + } else { -+ flags = 0; -+ extra = listp->head; /* subdirs inherit our rules */ ++ flags = listp->parent->match_flags & MATCHFLG_INCLUDE ++ ? XFLG_DEF_INCLUDE : 0; ++ extra = listp->head; /* Subdirs inherit our rules. */ + } + clear_exclude_list(listp, extra); + if (strlcpy(fname + offset, file, MAXPATHLEN - offset) -+ < MAXPATHLEN - offset) { ++ < MAXPATHLEN - offset) + add_exclude_file(listp, fname, flags); -+ } else { ++ else { + io_error |= IOERR_GENERAL; + rprintf(FINFO, + "cannot add local excludes in long-named directory %s\n", @@ -223,23 +237,28 @@ supersede your general rules instead of being subservient to them. + } + } + -+ return (void *) mem; ++ return (void*)push; +} + +void pop_local_excludes(void *mem) +{ + int i; -+ for (i = 0; i < local_exclude_list_cnt; i++) { -+ struct exclude_list_struct *listp = &local_exclude_lists[i]; ++ ++ for (i = 0; i < local_lists.cnt; i++) { ++ struct exclude_list_struct *listp = &local_lists.head[i]; ++ if (verbose > 2) { ++ rprintf(FINFO, "[%s] popping %sexclude list\n", ++ who_am_i(), listp->debug_type); ++ } + free_exclude_list(listp); + } -+ memcpy(local_exclude_lists, mem, -+ sizeof (struct exclude_list_struct) * local_exclude_list_cnt); ++ free(local_lists.head); ++ local_lists = *(struct exclude_list_root*)mem; + free(mem); } static int check_one_exclude(char *name, struct exclude_struct *ex, -@@ -139,7 +248,8 @@ static int check_one_exclude(char *name, +@@ -139,7 +265,8 @@ static int check_one_exclude(char *name, if (!name[0]) return 0; @@ -249,7 +268,7 @@ supersede your general rules instead of being subservient to them. if (*pattern == '/') { match_start = 1; -@@ -206,9 +316,11 @@ static void report_exclude_result(char c +@@ -206,9 +333,11 @@ static void report_exclude_result(char c if (verbose >= 2) { rprintf(FINFO, "[%s] %scluding %s %s because of %spattern %s%s\n", @@ -263,13 +282,13 @@ supersede your general rules instead of being subservient to them. } } -@@ -223,10 +335,18 @@ int check_exclude(struct exclude_list_st +@@ -222,10 +351,18 @@ int check_exclude(struct exclude_list_st struct exclude_struct *ent; for (ent = listp->head; ent; ent = ent->next) { -+ if (ent->match_flags & MATCHFLG_INSERT_FILE) { ++ if (ent->match_flags & MATCHFLG_MERGE_FILE) { + struct exclude_list_struct *lp -+ = &local_exclude_lists[ent->slash_cnt]; ++ = &local_lists.head[ent->slash_cnt]; + int rc = check_exclude(lp, name, name_is_dir); + if (rc) + return rc; @@ -283,7 +302,7 @@ supersede your general rules instead of being subservient to them. } } -@@ -242,11 +362,11 @@ int check_exclude(struct exclude_list_st +@@ -241,11 +378,11 @@ int check_exclude(struct exclude_list_st * *incl_ptr value will be 1 for an include, 0 for an exclude, and -1 for * the list-clearing "!" token. */ @@ -297,7 +316,7 @@ supersede your general rules instead of being subservient to them. if (xflags & XFLG_WORD_SPLIT) { /* Skip over any initial whitespace. */ -@@ -256,13 +376,19 @@ static const char *get_exclude_tok(const +@@ -255,13 +392,19 @@ static const char *get_exclude_tok(const p = (const char *)s; } @@ -310,7 +329,7 @@ supersede your general rules instead of being subservient to them. + if (*s == '+') + mflags |= MATCHFLG_INCLUDE; + else if (*s == '.') { -+ mflags |= MATCHFLG_INSERT_FILE; ++ mflags |= MATCHFLG_MERGE_FILE; + if (xflags & XFLG_DEF_INCLUDE) + mflags |= MATCHFLG_INCLUDE; + } @@ -322,7 +341,7 @@ supersede your general rules instead of being subservient to them. if (xflags & XFLG_WORD_SPLIT) { const unsigned char *cp = s; -@@ -274,9 +400,10 @@ static const char *get_exclude_tok(const +@@ -273,9 +416,10 @@ static const char *get_exclude_tok(const len = strlen(s); if (*p == '!' && len == 1 && !(xflags & XFLG_NO_PREFIXES)) @@ -334,7 +353,7 @@ supersede your general rules instead of being subservient to them. return (const char *)s; } -@@ -284,7 +411,7 @@ static const char *get_exclude_tok(const +@@ -283,7 +427,7 @@ static const char *get_exclude_tok(const void add_exclude(struct exclude_list_struct *listp, const char *pattern, int xflags) { @@ -343,7 +362,7 @@ supersede your general rules instead of being subservient to them. const char *cp; if (!pattern) -@@ -293,22 +420,44 @@ void add_exclude(struct exclude_list_str +@@ -292,22 +436,48 @@ void add_exclude(struct exclude_list_str cp = pattern; pat_len = 0; while (1) { @@ -351,26 +370,30 @@ supersede your general rules instead of being subservient to them. + cp = get_exclude_tok(cp + pat_len, &pat_len, &mflags, xflags); if (!pat_len) break; - /* If we got the special "!" token, clear the list. */ +- /* If we got the special "!" token, clear the list. */ - if (incl < 0) -+ if (mflags & MATCHFLG_CLEAR_LIST) { - free_exclude_list(listp); +- free_exclude_list(listp); - else { - make_exclude(listp, cp, pat_len, incl); - -- if (verbose > 2) { ++ if (mflags & MATCHFLG_CLEAR_LIST) { + if (verbose > 2) { - rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s)\n", - who_am_i(), pat_len, cp, - listp->debug_type, - incl ? "include" : "exclude"); ++ rprintf(FINFO, "[%s] clearing %sexclude list\n", ++ who_am_i(), listp->debug_type); ++ } ++ free_exclude_list(listp); + continue; + } -+ if (mflags & MATCHFLG_INSERT_FILE) { ++ if (mflags & MATCHFLG_MERGE_FILE) { + char name[MAXPATHLEN]; + if ((unsigned) pat_len >= sizeof name) -+ continue; // XXX complain? ++ continue; /* XXX complain? */ + strlcpy(name, cp, pat_len+1); -+ if (listp->parent || strchr(name, '/') != NULL) { ++ if (strchr(name, '/') != NULL) { + if (sanitize_paths) + sanitize_path(name, curr_dir); + if (*name == '/') @@ -379,7 +402,7 @@ supersede your general rules instead of being subservient to them. + if (strlcpy(dirbuf + dirbuf_offset, + name, MAXPATHLEN - dirbuf_offset) + >= MAXPATHLEN - dirbuf_offset) -+ continue; // XXX complain? ++ continue; /* XXX complain? */ + cp = dirbuf; + } + add_exclude_file(listp, cp, @@ -387,18 +410,19 @@ supersede your general rules instead of being subservient to them. + continue; } } ++ + make_exclude(listp, cp, pat_len, mflags); + + if (verbose > 2) { + rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n", + who_am_i(), pat_len, cp, listp->debug_type, -+ mflags & MATCHFLG_INSERT_FILE ? "FILE " : "", ++ mflags & MATCHFLG_MERGE_FILE ? "FILE " : "", + mflags & MATCHFLG_INCLUDE ? "in" : "ex"); + } } } -@@ -384,15 +533,19 @@ void send_exclude_list(int f) +@@ -383,15 +553,19 @@ void send_exclude_list(int f) l = strlcpy(p, ent->pattern, sizeof p); if (l == 0 || l >= MAXPATHLEN) continue; @@ -413,7 +437,7 @@ supersede your general rules instead of being subservient to them. write_int(f, l + 2); write_buf(f, "+ ", 2); - } else if ((*p == '-' || *p == '+') && p[1] == ' ') { -+ } else if (ent->match_flags & MATCHFLG_INSERT_FILE) { ++ } else if (ent->match_flags & MATCHFLG_MERGE_FILE) { + write_int(f, l + 2); + write_buf(f, ". ", 2); + } else if ((*p == '-' || *p == '+' || *p == '.') @@ -421,7 +445,7 @@ supersede your general rules instead of being subservient to them. write_int(f, l + 2); write_buf(f, "- ", 2); } else -@@ -433,6 +586,7 @@ void add_cvs_excludes(void) +@@ -432,6 +606,7 @@ void add_cvs_excludes(void) char fname[MAXPATHLEN]; char *p; @@ -430,7 +454,7 @@ supersede your general rules instead of being subservient to them. XFLG_WORD_SPLIT | XFLG_NO_PREFIXES); --- flist.c 22 Apr 2004 22:17:15 -0000 1.216 -+++ flist.c 22 Apr 2004 23:45:51 -0000 ++++ flist.c 24 Apr 2004 18:25:55 -0000 @@ -39,8 +39,6 @@ extern int module_id; extern int ignore_errors; extern int numeric_ids; @@ -519,7 +543,7 @@ supersede your general rules instead of being subservient to them. closedir(d); } --- proto.h 22 Apr 2004 09:58:09 -0000 1.189 -+++ proto.h 22 Apr 2004 23:45:51 -0000 ++++ proto.h 24 Apr 2004 18:25:55 -0000 @@ -51,7 +51,8 @@ int start_daemon(int f_in, int f_out); int daemon_main(void); void setup_protocol(int f_out,int f_in); @@ -531,7 +555,7 @@ supersede your general rules instead of being subservient to them. void add_exclude(struct exclude_list_struct *listp, const char *pattern, int xflags); --- rsync.h 22 Apr 2004 09:58:24 -0000 1.198 -+++ rsync.h 22 Apr 2004 23:45:52 -0000 ++++ rsync.h 24 Apr 2004 18:25:57 -0000 @@ -490,18 +490,21 @@ struct map_struct { #define MATCHFLG_WILD2 (1<<1) /* pattern has '**' */ #define MATCHFLG_WILD2_PREFIX (1<<2) /* pattern starts with '**' */ @@ -539,7 +563,7 @@ supersede your general rules instead of being subservient to them. +#define MATCHFLG_INCLUDE (1<<4) /* this is an include, not an exclude */ +#define MATCHFLG_CLEAR_LIST (1<<5) /* this item is the "!" token */ +#define MATCHFLG_DIRECTORY (1<<6) /* this matches only directories */ -+#define MATCHFLG_INSERT_FILE (1<<7) /* specifies a file to insert */ ++#define MATCHFLG_MERGE_FILE (1<<7) /* specifies a file to merge */ +#define MATCHFLG_CVSIGNORE (1<<8) /* parse this as a .cvsignore file */ struct exclude_struct { struct exclude_struct *next; @@ -558,3 +582,78 @@ supersede your general rules instead of being subservient to them. char *debug_type; }; +--- rsync.yo 24 Apr 2004 06:16:04 -0000 1.164 ++++ rsync.yo 24 Apr 2004 18:25:57 -0000 +@@ -1064,6 +1064,72 @@ itemize( + it would be excluded by the "*") + ) + ++manpagesection(MERGING EXCLUDE FILES) ++ ++You can merge whole files into an exclude file using a rule that starts ++with a ". " (a dot followed by a space) and has a filename in place of the ++pattern. There are two types of merge rules, single-instance and ++per-directory: ++ ++itemize( ++ it() If the filename has no slashes in it, it is a per-directory merge; ++ rsync scans every directory that is traversed and merges the named file's ++ contents (when it exists), putting the contents of each subdirectory's ++ file at the start of this per-directory sub-list (so subdirectories ++ inherit the contents of their parent directories by default, but each ++ subdirectory's rules have precedence over the parent's rules). ++ ++ it() If a filename has a slash in it, it is a single-instance merge; the ++ named file's contents will be merged into the current exclude file, ++ replacing the merge rule. Thus, you should use the name ./foo instead of ++ foo if you don't want to scan for "foo" in all the subdirectories of the ++ current directory. ++) ++ ++Note also that you can eliminate all the inherited rules for the current ++per-directory ruleset by putting the list-clearing token (!) in the file. ++This clears only the rules of the current per-directory sub-list (up ++through the token) and only for the current directory and its ++subdirectories. ++ ++Here's an example. Specify the file that holds this set of rules via a ++normal --exclude-from option: ++ ++verb( ++ . /home/user/.global_excludes ++ - *.gz ++ . .excl ++ + *.[ch] ++ - *.o ++) ++ ++This will merge the contents of the /home/user/.global_excludes file at the ++start of the list and also turns the ".excl" filename into a per-directory ++exclude file whose local contents will be merged into the list in place of ++the .excl line. ++ ++Additionally, you can affect where the --cvs-exclude (-C) option's ++inclusion of a per-directory .cvsignore file gets placed into your rules by ++adding an explicit a merge rule for ".cvsignore". For instance, specifying ++this: ++ ++verb( ++ rsync -avC --exclude='. .cvsignore' --exclude-from=foo a/ b ++) ++ ++will merge all the per-directory .cvsignore rules at the start of your list ++rather than at the end. This allows their dir-specific rules to supersede ++your rules instead of being subservient to them. (The global rules taken ++from the $HOME/.cvsignore file and from $CVSIGNORE are not affected by ++this.) ++ ++Note also that the parsing of any merge-file named ".cvsignore" is always ++done in a CVS-compatible manner (even if -C wasn't specified) -- i.e. the ++rules are always exclude rules (even when specified by an include option), ++they are split on whitespace, no special prefixes or list-clearing tokens ++are honored, and (for per-directory files) subdirectories don't inherit the ++parent directory's rules. ++ + manpagesection(BATCH MODE) + + bf(Note:) Batch mode should be considered experimental in this version -- 2.34.1