1 After applying this patch and running configure, you MUST run this
6 This patch adds the ability to merge rules into your excludes/includes
7 using a ". FILE" idiom. If you specify a name without slashes, that
8 filename will be looked for in every subdirectory that rsync visits,
9 and its rules will affect the current directory and its subdirectories.
13 rsync -av --exclude='. .excl' from/ to
15 The above will look for a file named ".excl" in every directory of the
16 hierarchy that rsync visits, and it will exclude (by default) names
17 based on the rules found therein. If one of the .excl files contains
25 Then the file ".excl2" will also be read in the current dir, and all
26 subdirs of the current dir. The file ".excl3" would just be read in
27 for the current dir because its name contained a slash.
31 --- exclude.c 14 May 2004 21:23:41 -0000 1.76
32 +++ exclude.c 15 May 2004 07:01:28 -0000
33 @@ -30,32 +30,89 @@ extern int verbose;
38 +extern int sanitize_paths;
40 extern char curr_dir[];
42 -struct exclude_list_struct exclude_list = { 0, 0, "" };
43 -struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " };
44 -struct exclude_list_struct server_exclude_list = { 0, 0, "server " };
45 +struct exclude_list_struct exclude_list = { 0, 0, 0, "" };
46 +struct exclude_list_struct server_exclude_list = { 0, 0, 0, "server " };
47 char *exclude_path_prefix = NULL;
49 -/** Build an exclude structure given a exclude pattern */
50 -static void make_exclude(struct exclude_list_struct *listp, const char *pattern,
51 - int pat_len, int include)
52 +struct exclude_list_root {
53 + struct exclude_list_struct *array;
57 +static char dirbuf[MAXPATHLEN];
58 +static unsigned int dirbuf_offset = 0;
60 +/* Each exclude_list_struct describes a singly-linked list by keeping track
61 + * of both the head and tail pointers. The list is slightly unusual in that
62 + * a parent-dir's content can be appended to the end of the local list in a
63 + * special way: the last item in the local list has its "next" pointer set
64 + * to point to the parent's list, but the local list's tail pointer points
65 + * at the end of the local list. Thus, if the local list is empty, the head
66 + * will be pointing at the parent content but the tail will be NULL. To
67 + * help you visualize this, here are the possible list arrangements:
69 + * Completely Empty Local Content Only
70 + * ================================== ====================================
71 + * head -> NULL head -> Local1 -> Local2 -> NULL
72 + * tail -> NULL tail -------------^
74 + * Inherited Content Only Both Local and Inherited Content
75 + * ================================== ====================================
76 + * head -> Parent1 -> Parent2 -> NULL head -> L1 -> L2 -> P1 -> P2 -> NULL
77 + * tail -> NULL tail ---------^
79 + * This means that anyone wanting to traverse the whole list to USE it just
80 + * needs to start at the head and use the "next" pointers until it goes
81 + * NULL. To add new local content, we insert the item after the tail item
82 + * and update the tail (obviously, if "tail" was NULL, we insert it at the
83 + * head). To clear the local list, WE MUST NOT FREE THE INHERITED CONTENT
84 + * because it is shared between the current list and all our parent list(s).
85 + * The easiest way to avoid this is to simply truncate the list after the
86 + * tail item and free the local list from the head. When inheriting the
87 + * list for a new local dir, we just save off the exclude_list_struct values
88 + * and set the tail to NULL.
91 +/** Build an exclude structure given an exclude pattern. */
92 +static void make_exclude(struct exclude_list_struct *listp, const char *pat,
93 + unsigned int pat_len, int mflags)
95 struct exclude_struct *ret;
98 + unsigned int ex_len;
100 + if (mflags & MATCHFLG_MERGE_FILE) {
101 + struct exclude_struct *ex;
102 + /* If the local include file was already mentioned, don't
104 + for (ex = listp->head; ex; ex = ex->next) {
105 + if ((ex->match_flags & MATCHFLG_MERGE_FILE)
106 + && strlen(ex->pattern) == pat_len
107 + && strncmp(ex->pattern, pat, pat_len) == 0)
110 + if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/'))
111 + && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) {
112 + mflags |= MATCHFLG_CVSIGNORE;
113 + mflags &= ~MATCHFLG_INCLUDE;
115 + mflags &= ~MATCHFLG_CVSIGNORE;
118 ret = new(struct exclude_struct);
120 out_of_memory("make_exclude");
122 memset(ret, 0, sizeof ret[0]);
123 - ret->include = include;
125 if (exclude_path_prefix)
126 - ret->match_flags |= MATCHFLG_ABS_PATH;
127 - if (exclude_path_prefix && *pattern == '/')
128 + mflags |= MATCHFLG_ABS_PATH;
129 + if (exclude_path_prefix && *pat == '/')
130 ex_len = strlen(exclude_path_prefix);
133 @@ -64,33 +121,52 @@ static void make_exclude(struct exclude_
134 out_of_memory("make_exclude");
136 memcpy(ret->pattern, exclude_path_prefix, ex_len);
137 - strlcpy(ret->pattern + ex_len, pattern, pat_len + 1);
138 + strlcpy(ret->pattern + ex_len, pat, pat_len + 1);
141 if (strpbrk(ret->pattern, "*[?")) {
142 - ret->match_flags |= MATCHFLG_WILD;
143 + mflags |= MATCHFLG_WILD;
144 if ((cp = strstr(ret->pattern, "**")) != NULL) {
145 - ret->match_flags |= MATCHFLG_WILD2;
146 + mflags |= MATCHFLG_WILD2;
147 /* If the pattern starts with **, note that. */
148 if (cp == ret->pattern)
149 - ret->match_flags |= MATCHFLG_WILD2_PREFIX;
150 + mflags |= MATCHFLG_WILD2_PREFIX;
154 if (pat_len > 1 && ret->pattern[pat_len-1] == '/') {
155 ret->pattern[pat_len-1] = 0;
156 - ret->directory = 1;
157 + mflags |= MATCHFLG_DIRECTORY;
160 for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++)
164 + if (!listp->tail) {
165 + ret->next = listp->head;
166 listp->head = listp->tail = ret;
169 + ret->next = listp->tail->next;
170 listp->tail->next = ret;
174 + if (mflags & MATCHFLG_MERGE_FILE) {
175 + struct exclude_list_struct *lp;
176 + int ndx = local_lists.cnt++;
177 + local_lists.array = realloc_array(local_lists.array,
178 + struct exclude_list_struct, local_lists.cnt);
179 + if (!local_lists.array)
180 + out_of_memory("make_exclude");
181 + lp = &local_lists.array[ndx];
182 + lp->head = lp->tail = NULL;
183 + if (asprintf(&lp->debug_type, "per-dir %s ", ret->pattern) < 0)
184 + out_of_memory("make_exclude");
186 + ret->slash_cnt = ndx;
189 + ret->match_flags = mflags;
192 static void free_exclude(struct exclude_struct *ex)
193 @@ -99,18 +175,90 @@ static void free_exclude(struct exclude_
197 -void free_exclude_list(struct exclude_list_struct *listp)
198 +static void free_exclude_list(struct exclude_list_struct *listp)
200 - struct exclude_struct *ent, *next;
202 - for (ent = listp->head; ent; ent = next) {
206 + struct exclude_struct *ent, *next;
207 + /* Truncate any extra parent items from local list. */
208 + listp->tail->next = NULL;
209 + for (ent = listp->head; ent; ent = next) {
215 listp->head = listp->tail = NULL;
218 +void *push_local_excludes(char *fname, unsigned int offset)
221 + struct exclude_list_root *push = new_array(struct exclude_list_root, 1);
224 + out_of_memory("push_local_excludes");
226 + push->cnt = local_lists.cnt;
227 + push->array = new_array(struct exclude_list_struct, local_lists.cnt);
229 + out_of_memory("push_local_excludes");
231 + memcpy(push->array, local_lists.array,
232 + sizeof (struct exclude_list_struct) * local_lists.cnt);
234 + /* Make it easy to construct the full path for a merge-file that was
235 + * specified with a relative path by saving off the current dir. */
236 + memcpy(dirbuf, fname, offset);
237 + dirbuf_offset = offset;
239 + for (i = 0; i < local_lists.cnt; i++) {
240 + struct exclude_list_struct *listp = &local_lists.array[i];
241 + char *file = listp->parent->pattern;
245 + rprintf(FINFO, "[%s] pushing %sexclude list\n",
246 + who_am_i(), listp->debug_type);
248 + if (listp->parent->match_flags & MATCHFLG_CVSIGNORE) {
249 + flags = XFLG_WORD_SPLIT | XFLG_WORDS_ONLY;
250 + listp->head = NULL; /* Subdirs don't get our rules. */
252 + flags = listp->parent->match_flags & MATCHFLG_INCLUDE
253 + ? XFLG_DEF_INCLUDE : 0;
255 + listp->tail = NULL;
256 + if (strlcpy(fname + offset, file, MAXPATHLEN - offset)
257 + < MAXPATHLEN - offset)
258 + add_exclude_file(listp, fname, flags);
260 + io_error |= IOERR_GENERAL;
262 + "cannot add local excludes in long-named directory %s\n",
263 + full_fname(fname));
267 + return (void*)push;
270 +void pop_local_excludes(void *mem)
274 + for (i = 0; i < local_lists.cnt; i++) {
275 + struct exclude_list_struct *listp = &local_lists.array[i];
277 + rprintf(FINFO, "[%s] popping %sexclude list\n",
278 + who_am_i(), listp->debug_type);
280 + free_exclude_list(listp);
282 + free(local_lists.array);
283 + local_lists = *(struct exclude_list_root*)mem;
287 static int check_one_exclude(char *name, struct exclude_struct *ex,
290 @@ -134,7 +282,8 @@ static int check_one_exclude(char *name,
292 if (!name[0]) return 0;
294 - if (ex->directory && !name_is_dir) return 0;
295 + if ((ex->match_flags & MATCHFLG_DIRECTORY) && !name_is_dir)
298 if (*pattern == '/') {
300 @@ -201,9 +350,11 @@ static void report_exclude_result(char c
303 rprintf(FINFO, "[%s] %scluding %s %s because of %spattern %s%s\n",
304 - who_am_i(), ent->include ? "in" : "ex",
306 + ent->match_flags & MATCHFLG_INCLUDE ? "in" : "ex",
307 name_is_dir ? "directory" : "file", name, type,
308 - ent->pattern, ent->directory ? "/" : "");
310 + ent->match_flags & MATCHFLG_DIRECTORY ? "/" : "");
314 @@ -217,10 +368,18 @@ int check_exclude(struct exclude_list_st
315 struct exclude_struct *ent;
317 for (ent = listp->head; ent; ent = ent->next) {
318 + if (ent->match_flags & MATCHFLG_MERGE_FILE) {
319 + struct exclude_list_struct *lp
320 + = &local_lists.array[ent->slash_cnt];
321 + int rc = check_exclude(lp, name, name_is_dir);
326 if (check_one_exclude(name, ent, name_is_dir)) {
327 report_exclude_result(name, ent, name_is_dir,
329 - return ent->include ? 1 : -1;
330 + return (ent->match_flags & MATCHFLG_INCLUDE) ? 1 : -1;
334 @@ -236,11 +395,11 @@ int check_exclude(struct exclude_list_st
335 * *incl_ptr value will be 1 for an include, 0 for an exclude, and -1 for
336 * the list-clearing "!" token.
338 -static const char *get_exclude_tok(const char *p, int *len_ptr, int *incl_ptr,
339 +static const char *get_exclude_tok(const char *p, int *len_ptr, int *flag_ptr,
342 const unsigned char *s = (const unsigned char *)p;
344 + int len, mflags = 0;
346 if (xflags & XFLG_WORD_SPLIT) {
347 /* Skip over any initial whitespace. */
348 @@ -250,13 +409,19 @@ static const char *get_exclude_tok(const
352 - /* Is this a '+' or '-' followed by a space (not whitespace)? */
353 + /* Is this a +/-/. followed by a space (not whitespace)? */
354 if (!(xflags & XFLG_WORDS_ONLY)
355 - && (*s == '-' || *s == '+') && s[1] == ' ') {
356 - *incl_ptr = *s == '+';
357 + && (*s == '-' || *s == '+' || *s == '.') && s[1] == ' ') {
359 + mflags |= MATCHFLG_INCLUDE;
360 + else if (*s == '.') {
361 + mflags |= MATCHFLG_MERGE_FILE;
362 + if (xflags & XFLG_DEF_INCLUDE)
363 + mflags |= MATCHFLG_INCLUDE;
367 - *incl_ptr = xflags & XFLG_DEF_INCLUDE;
368 + } else if (xflags & XFLG_DEF_INCLUDE)
369 + mflags |= MATCHFLG_INCLUDE;
371 if (xflags & XFLG_WORD_SPLIT) {
372 const unsigned char *cp = s;
373 @@ -268,9 +433,10 @@ static const char *get_exclude_tok(const
376 if (*p == '!' && len == 1 && !(xflags & XFLG_WORDS_ONLY))
378 + mflags |= MATCHFLG_CLEAR_LIST;
381 + *flag_ptr = mflags;
382 return (const char *)s;
385 @@ -278,7 +444,7 @@ static const char *get_exclude_tok(const
386 void add_exclude(struct exclude_list_struct *listp, const char *pattern,
390 + int pat_len, mflags;
394 @@ -287,27 +453,49 @@ void add_exclude(struct exclude_list_str
398 - cp = get_exclude_tok(cp + pat_len, &pat_len, &incl, xflags);
399 + cp = get_exclude_tok(cp + pat_len, &pat_len, &mflags, xflags);
402 - /* If we got the special "!" token, clear the list. */
404 + if (mflags & MATCHFLG_CLEAR_LIST) {
407 "[%s] clearing %sexclude list\n",
408 who_am_i(), listp->debug_type);
410 free_exclude_list(listp);
412 - make_exclude(listp, cp, pat_len, incl);
415 - rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s)\n",
416 - who_am_i(), pat_len, cp,
418 - incl ? "include" : "exclude");
421 + if (mflags & MATCHFLG_MERGE_FILE) {
422 + char name[MAXPATHLEN];
423 + if ((unsigned) pat_len >= sizeof name)
424 + continue; /* XXX complain? */
425 + strlcpy(name, cp, pat_len+1);
426 + if (strchr(name, '/') != NULL) {
427 + if (sanitize_paths)
428 + sanitize_path(name, curr_dir);
432 + if (strlcpy(dirbuf + dirbuf_offset,
433 + name, MAXPATHLEN - dirbuf_offset)
434 + >= MAXPATHLEN - dirbuf_offset)
435 + continue; /* XXX complain? */
438 + add_exclude_file(listp, cp,
439 + xflags | XFLG_FATAL_ERRORS);
444 + make_exclude(listp, cp, pat_len, mflags);
447 + rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n",
448 + who_am_i(), pat_len, cp, listp->debug_type,
449 + mflags & MATCHFLG_MERGE_FILE ? "FILE " : "",
450 + mflags & MATCHFLG_INCLUDE ? "in" : "ex");
455 @@ -383,15 +571,19 @@ void send_exclude_list(int f)
456 l = strlcpy(p, ent->pattern, sizeof p);
457 if (l == 0 || l >= MAXPATHLEN)
459 - if (ent->directory) {
460 + if (ent->match_flags & MATCHFLG_DIRECTORY) {
465 - if (ent->include) {
466 + if (ent->match_flags & MATCHFLG_INCLUDE) {
468 write_buf(f, "+ ", 2);
469 - } else if ((*p == '-' || *p == '+') && p[1] == ' ') {
470 + } else if (ent->match_flags & MATCHFLG_MERGE_FILE) {
471 + write_int(f, l + 2);
472 + write_buf(f, ". ", 2);
473 + } else if ((*p == '-' || *p == '+' || *p == '.')
476 write_buf(f, "- ", 2);
478 @@ -432,6 +624,7 @@ void add_cvs_excludes(void)
479 char fname[MAXPATHLEN];
482 + add_exclude(&exclude_list, ". .cvsignore", 0);
483 add_exclude(&exclude_list, default_cvsignore,
484 XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
486 --- flist.c 14 May 2004 21:23:41 -0000 1.222
487 +++ flist.c 15 May 2004 07:01:28 -0000
488 @@ -39,8 +39,6 @@ extern int module_id;
489 extern int ignore_errors;
490 extern int numeric_ids;
492 -extern int cvs_exclude;
495 extern char curr_dir[MAXPATHLEN];
496 extern char *files_from;
497 @@ -65,7 +63,6 @@ extern int write_batch;
499 extern struct exclude_list_struct exclude_list;
500 extern struct exclude_list_struct server_exclude_list;
501 -extern struct exclude_list_struct local_exclude_list;
505 @@ -210,8 +207,6 @@ int link_stat(const char *path, STRUCT_S
507 static int check_exclude_file(char *fname, int is_dir, int exclude_level)
511 #if 0 /* This currently never happens, so avoid a useless compare. */
512 if (exclude_level == NO_EXCLUDES)
514 @@ -233,10 +228,7 @@ static int check_exclude_file(char *fnam
515 if (exclude_level != ALL_EXCLUDES)
517 if (exclude_list.head
518 - && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0)
520 - if (local_exclude_list.head
521 - && check_exclude(&local_exclude_list, fname, is_dir) < 0)
522 + && check_exclude(&exclude_list, fname, is_dir) < 0)
526 @@ -946,15 +938,7 @@ void send_file_name(int f, struct file_l
528 if (recursive && S_ISDIR(file->mode)
529 && !(file->flags & FLAG_MOUNT_POINT)) {
530 - struct exclude_list_struct last_list = local_exclude_list;
531 - local_exclude_list.head = local_exclude_list.tail = NULL;
532 send_directory(f, flist, f_name_to(file, fbuf));
534 - rprintf(FINFO, "[%s] popping %sexclude list\n",
535 - who_am_i(), local_exclude_list.debug_type);
537 - free_exclude_list(&local_exclude_list);
538 - local_exclude_list = last_list;
542 @@ -965,6 +949,7 @@ static void send_directory(int f, struct
544 char fname[MAXPATHLEN];
546 + void *save_excludes;
550 @@ -989,18 +974,7 @@ static void send_directory(int f, struct
555 - if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset)
556 - < MAXPATHLEN - offset) {
557 - add_exclude_file(&local_exclude_list, fname,
558 - XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
560 - io_error |= IOERR_GENERAL;
562 - "cannot cvs-exclude in long-named directory %s\n",
563 - full_fname(fname));
566 + save_excludes = push_local_excludes(fname, offset);
568 for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
569 char *dname = d_name(di);
570 @@ -1021,6 +995,8 @@ static void send_directory(int f, struct
571 rprintf(FERROR, "readdir(%s): (%d) %s\n",
572 dir, errno, strerror(errno));
575 + pop_local_excludes(save_excludes);
579 --- rsync.h 13 May 2004 18:51:22 -0000 1.203
580 +++ rsync.h 15 May 2004 07:01:29 -0000
581 @@ -493,18 +493,21 @@ struct map_struct {
582 #define MATCHFLG_WILD2 (1<<1) /* pattern has '**' */
583 #define MATCHFLG_WILD2_PREFIX (1<<2) /* pattern starts with '**' */
584 #define MATCHFLG_ABS_PATH (1<<3) /* path-match on absolute path */
585 +#define MATCHFLG_INCLUDE (1<<4) /* this is an include, not an exclude */
586 +#define MATCHFLG_CLEAR_LIST (1<<5) /* this item is the "!" token */
587 +#define MATCHFLG_DIRECTORY (1<<6) /* this matches only directories */
588 +#define MATCHFLG_MERGE_FILE (1<<7) /* specifies a file to merge */
589 +#define MATCHFLG_CVSIGNORE (1<<8) /* parse this as a .cvsignore file */
590 struct exclude_struct {
591 struct exclude_struct *next;
599 struct exclude_list_struct {
600 - struct exclude_struct *head;
601 - struct exclude_struct *tail;
602 + struct exclude_struct *head, *tail;
603 + struct exclude_struct *parent;
607 --- rsync.yo 7 May 2004 00:18:37 -0000 1.169
608 +++ rsync.yo 15 May 2004 07:01:29 -0000
609 @@ -1075,6 +1075,72 @@ itemize(
610 it would be excluded by the "*")
613 +manpagesection(MERGING EXCLUDE FILES)
615 +You can merge whole files into an exclude file using a rule that starts
616 +with a ". " (a dot followed by a space) and has a filename in place of the
617 +pattern. There are two types of merge rules, single-instance and
621 + it() If the filename has no slashes in it, it is a per-directory merge;
622 + rsync scans every directory that is traversed and merges the named file's
623 + contents (when it exists), putting the contents of each subdirectory's
624 + file at the start of this per-directory sub-list (so subdirectories
625 + inherit the contents of their parent directories by default, but each
626 + subdirectory's rules have precedence over the parent's rules).
628 + it() If a filename has a slash in it, it is a single-instance merge; the
629 + named file's contents will be merged into the current exclude file,
630 + replacing the merge rule. Thus, you should use the name ./foo instead of
631 + foo if you don't want to scan for "foo" in all the subdirectories of the
635 +Note also that you can eliminate all the inherited rules for the current
636 +per-directory ruleset by putting the list-clearing token (!) in the file.
637 +This clears only the rules of the current per-directory sub-list (up
638 +through the token) and only for the current directory and its
641 +Here's an example. Specify the file that holds this set of rules via a
642 +normal --exclude-from option:
645 + . /home/user/.global_excludes
652 +This will merge the contents of the /home/user/.global_excludes file at the
653 +start of the list and also turns the ".excl" filename into a per-directory
654 +exclude file whose local contents will be merged into the list in place of
657 +Additionally, you can affect where the --cvs-exclude (-C) option's
658 +inclusion of a per-directory .cvsignore file gets placed into your rules by
659 +adding an explicit a merge rule for ".cvsignore". For instance, specifying
663 + rsync -avC --exclude='. .cvsignore' --exclude-from=foo a/ b
666 +will merge all the per-directory .cvsignore rules at the start of your list
667 +rather than at the end. This allows their dir-specific rules to supersede
668 +your rules instead of being subservient to them. (The global rules taken
669 +from the $HOME/.cvsignore file and from $CVSIGNORE are not affected by
672 +Note also that the parsing of any merge-file named ".cvsignore" is always
673 +done in a CVS-compatible manner (even if -C wasn't specified) -- i.e. the
674 +rules are always exclude rules (even when specified by an include option),
675 +they are split on whitespace, no special prefixes or list-clearing tokens
676 +are honored, and (for per-directory files) subdirectories don't inherit the
677 +parent directory's rules.
679 manpagesection(BATCH MODE)
681 bf(Note:) Batch mode should be considered experimental in this version