1 After applying this patch and running configure, you MUST run this
6 This patch adds the ability to merge rules into your excludes/includes
7 using a ". FILE" idiom. If you specify a name without slashes, that
8 filename will be looked for in every subdirectory that rsync visits,
9 and the rules found in that subdirectory's file will affect that dir
10 and its subdirectories.
14 rsync -av --exclude='. .excl' from/ to
16 The above will look for a file named ".excl" in every directory of the
17 hierarchy that rsync visits, and it will exclude (by default) names
18 based on the rules found therein. If one of the .excl files contains
26 Then the file ".excl2" will also be read in the current dir, and all
27 subdirs of the current dir. The file ".excl3" would just be read in
28 for the current dir because its name contained a slash.
32 --- exclude.c 22 May 2004 05:32:20 -0000 1.82
33 +++ exclude.c 24 May 2004 00:44:00 -0000
34 @@ -30,13 +30,56 @@ extern int verbose;
39 +extern int sanitize_paths;
41 extern char curr_dir[];
43 struct exclude_list_struct exclude_list = { 0, 0, "" };
44 -struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " };
45 struct exclude_list_struct server_exclude_list = { 0, 0, "server " };
46 char *exclude_path_prefix = NULL;
47 +struct exclude_struct **mergelist_parents;
48 +int mergelist_cnt = 0;
49 +int mergelist_size = 0;
51 +struct mergelist_save_struct {
52 + struct exclude_list_struct *array;
56 +static char dirbuf[MAXPATHLEN];
57 +static unsigned int dirbuf_offset = 0;
59 +/* Each exclude_list_struct describes a singly-linked list by keeping track
60 + * of both the head and tail pointers. The list is slightly unusual in that
61 + * a parent-dir's content can be appended to the end of the local list in a
62 + * special way: the last item in the local list has its "next" pointer set
63 + * to point to the inherited list, but the local list's tail pointer points
64 + * at the end of the local list. Thus, if the local list is empty, the head
65 + * will be pointing at the inherited content but the tail will be NULL. To
66 + * help you visualize this, here are the possible list arrangements:
68 + * Completely Empty Local Content Only
69 + * ================================== ====================================
70 + * head -> NULL head -> Local1 -> Local2 -> NULL
71 + * tail -> NULL tail -------------^
73 + * Inherited Content Only Both Local and Inherited Content
74 + * ================================== ====================================
75 + * head -> Parent1 -> Parent2 -> NULL head -> L1 -> L2 -> P1 -> P2 -> NULL
76 + * tail -> NULL tail ---------^
78 + * This means that anyone wanting to traverse the whole list to USE it just
79 + * needs to start at the head and use the "next" pointers until it goes
80 + * NULL. To add new local content, we insert the item after the tail item
81 + * and update the tail (obviously, if "tail" was NULL, we insert it at the
82 + * head). To clear the local list, WE MUST NOT FREE THE INHERITED CONTENT
83 + * because it is shared between the current list and our parent list(s).
84 + * The easiest way to handle this is to simply truncate the list after the
85 + * tail item and then free the local list from the head. When inheriting
86 + * the list for a new local dir, we just save off the exclude_list_struct
87 + * values (so we can pop back to them later) and set the tail to NULL.
90 /** Build an exclude structure given an exclude pattern. */
91 static void make_exclude(struct exclude_list_struct *listp, const char *pat,
92 @@ -46,6 +89,24 @@ static void make_exclude(struct exclude_
96 + if (mflags & MATCHFLG_MERGE_FILE) {
98 + /* If the local include file was already mentioned, don't
100 + for (i = 0; i < mergelist_cnt; i++) {
101 + struct exclude_struct *ex = mergelist_parents[i];
102 + if (strlen(ex->pattern) == pat_len
103 + && memcmp(ex->pattern, pat, pat_len) == 0)
106 + if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/'))
107 + && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) {
108 + mflags |= MATCHFLG_CVSIGNORE;
109 + mflags &= ~MATCHFLG_INCLUDE;
111 + mflags &= ~MATCHFLG_CVSIGNORE;
114 ret = new(struct exclude_struct);
116 out_of_memory("make_exclude");
117 @@ -81,14 +142,36 @@ static void make_exclude(struct exclude_
118 mflags |= MATCHFLG_DIRECTORY;
121 - for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++)
123 + if (mflags & MATCHFLG_MERGE_FILE) {
124 + struct exclude_list_struct *lp
125 + = new_array(struct exclude_list_struct, 1);
127 + out_of_memory("make_exclude");
128 + lp->head = lp->tail = NULL;
129 + if (asprintf(&lp->debug_type, "per-dir %s ", ret->pattern) < 0)
130 + out_of_memory("make_exclude");
131 + ret->u.mergelist = lp;
132 + if (mergelist_cnt == mergelist_size) {
133 + mergelist_size += 5;
134 + mergelist_parents = realloc_array(mergelist_parents,
135 + struct exclude_struct *,
137 + if (!mergelist_parents)
138 + out_of_memory("make_exclude");
140 + mergelist_parents[mergelist_cnt++] = ret;
142 + for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++)
143 + ret->u.slash_cnt++;
146 ret->match_flags = mflags;
149 + if (!listp->tail) {
150 + ret->next = listp->head;
151 listp->head = listp->tail = ret;
154 + ret->next = listp->tail->next;
155 listp->tail->next = ret;
158 @@ -96,22 +179,116 @@ static void make_exclude(struct exclude_
160 static void free_exclude(struct exclude_struct *ex)
162 + if (ex->match_flags & MATCHFLG_MERGE_FILE) {
163 + free(ex->u.mergelist->debug_type);
164 + free(ex->u.mergelist);
170 -void clear_exclude_list(struct exclude_list_struct *listp)
171 +static void clear_exclude_list(struct exclude_list_struct *listp)
173 - struct exclude_struct *ent, *next;
175 - for (ent = listp->head; ent; ent = next) {
179 + struct exclude_struct *ent, *next;
180 + /* Truncate any inherited items from the local list. */
181 + listp->tail->next = NULL;
182 + /* Now free everything that is left. */
183 + for (ent = listp->head; ent; ent = next) {
189 listp->head = listp->tail = NULL;
192 +void *push_local_excludes(char *fname, unsigned int offset)
194 + struct mergelist_save_struct *push;
195 + struct exclude_list_struct *ap;
198 + /* Make it easy to construct the full path for a merge-file that was
199 + * specified with a relative path by saving off the current dir. */
200 + memcpy(dirbuf, fname, offset);
201 + dirbuf_offset = offset;
203 + if (!(push = new_array(struct mergelist_save_struct, 1)))
204 + out_of_memory("push_local_excludes");
206 + push->count = mergelist_cnt;
207 + push->array = new_array(struct exclude_list_struct, mergelist_cnt);
209 + out_of_memory("push_local_excludes");
211 + for (i = 0, ap = push->array; i < mergelist_cnt; i++) {
212 + memcpy(ap++, mergelist_parents[i]->u.mergelist,
213 + sizeof (struct exclude_list_struct));
216 + /* Note: add_exclude_file() might increase mergelist_cnt, so keep
217 + * this loop separate from the above loop. */
218 + for (i = 0; i < mergelist_cnt; i++) {
219 + struct exclude_struct *ex = mergelist_parents[i];
220 + struct exclude_list_struct *lp = ex->u.mergelist;
224 + rprintf(FINFO, "[%s] pushing %sexclude list\n",
225 + who_am_i(), lp->debug_type);
228 + if (ex->match_flags & MATCHFLG_CVSIGNORE) {
229 + lp->head = NULL; /* CVS doesn't inherit rules. */
230 + flags = XFLG_WORD_SPLIT | XFLG_WORDS_ONLY;
232 + flags = ex->match_flags & MATCHFLG_INCLUDE
233 + ? XFLG_DEF_INCLUDE : 0;
235 + lp->tail = NULL; /* Switch any local rules to inherited. */
236 + if (strlcpy(dirbuf + dirbuf_offset, ex->pattern,
237 + MAXPATHLEN - dirbuf_offset) < MAXPATHLEN - dirbuf_offset)
238 + add_exclude_file(lp, dirbuf, flags);
240 + io_error |= IOERR_GENERAL;
242 + "cannot add local excludes in long-named directory %s\n",
243 + full_fname(dirbuf));
247 + return (void*)push;
250 +void pop_local_excludes(void *mem)
252 + struct mergelist_save_struct *pop = (struct mergelist_save_struct*)mem;
253 + struct exclude_list_struct *ap;
256 + for (i = mergelist_cnt; i-- > 0; ) {
257 + struct exclude_struct *ex = mergelist_parents[i];
258 + struct exclude_list_struct *lp = ex->u.mergelist;
261 + rprintf(FINFO, "[%s] popping %sexclude list\n",
262 + who_am_i(), lp->debug_type);
265 + clear_exclude_list(lp);
268 + mergelist_cnt = pop->count;
269 + for (i = 0, ap = pop->array; i < mergelist_cnt; i++) {
270 + memcpy(mergelist_parents[i]->u.mergelist, ap++,
271 + sizeof (struct exclude_list_struct));
278 static int check_one_exclude(char *name, struct exclude_struct *ex,
281 @@ -122,7 +299,7 @@ static int check_one_exclude(char *name,
282 /* If the pattern does not have any slashes AND it does not have
283 * a "**" (which could match a slash), then we just match the
284 * name portion of the path. */
285 - if (!ex->slash_cnt && !(ex->match_flags & MATCHFLG_WILD2)) {
286 + if (!ex->u.slash_cnt && !(ex->match_flags & MATCHFLG_WILD2)) {
287 if ((p = strrchr(name,'/')) != NULL)
290 @@ -148,9 +325,9 @@ static int check_one_exclude(char *name,
291 if (ex->match_flags & MATCHFLG_WILD) {
292 /* A non-anchored match with an infix slash and no "**"
293 * needs to match the last slash_cnt+1 name elements. */
294 - if (!match_start && ex->slash_cnt
295 + if (!match_start && ex->u.slash_cnt
296 && !(ex->match_flags & MATCHFLG_WILD2)) {
297 - int cnt = ex->slash_cnt + 1;
298 + int cnt = ex->u.slash_cnt + 1;
299 for (p = name + strlen(name) - 1; p >= name; p--) {
300 if (*p == '/' && !--cnt)
302 @@ -221,6 +398,13 @@ int check_exclude(struct exclude_list_st
303 struct exclude_struct *ent;
305 for (ent = listp->head; ent; ent = ent->next) {
306 + if (ent->match_flags & MATCHFLG_MERGE_FILE) {
307 + int rc = check_exclude(ent->u.mergelist, name,
313 if (check_one_exclude(name, ent, name_is_dir)) {
314 report_exclude_result(name, ent, name_is_dir,
316 @@ -253,11 +437,16 @@ static const char *get_exclude_tok(const
320 - /* Is this a '+' or '-' followed by a space (not whitespace)? */
321 + /* Is this a +/-/. followed by a space (not whitespace)? */
322 if (!(xflags & XFLG_WORDS_ONLY)
323 - && (*s == '-' || *s == '+') && s[1] == ' ') {
324 + && (*s == '-' || *s == '+' || *s == '.') && s[1] == ' ') {
326 mflags |= MATCHFLG_INCLUDE;
327 + else if (*s == '.') {
328 + mflags |= MATCHFLG_MERGE_FILE;
329 + if (xflags & XFLG_DEF_INCLUDE)
330 + mflags |= MATCHFLG_INCLUDE;
333 } else if (xflags & XFLG_DEF_INCLUDE)
334 mflags |= MATCHFLG_INCLUDE;
335 @@ -306,11 +495,52 @@ void add_exclude(struct exclude_list_str
339 + if (mflags & MATCHFLG_MERGE_FILE) {
340 + char name[MAXPATHLEN];
341 + if (pat_len >= sizeof name) {
343 + "merge filename too long: %s\n", cp);
346 + strlcpy(name, cp, pat_len+1);
347 + if (strchr(name, '/') != NULL) {
349 + if (*name == '/') {
350 + if (sanitize_paths) {
351 + mem = alloc_sanitize_path(name,
357 + if (sanitize_paths) {
358 + dirbuf[dirbuf_offset] = '\0';
359 + sanitize_path(name, dirbuf);
361 + if (strlcpy(dirbuf + dirbuf_offset,
362 + name, MAXPATHLEN - dirbuf_offset)
363 + >= MAXPATHLEN - dirbuf_offset) {
365 + "merge filename too long: %s...\n",
371 + add_exclude_file(listp, cp,
372 + xflags | XFLG_FATAL_ERRORS);
379 make_exclude(listp, cp, pat_len, mflags);
382 - rprintf(FINFO, "[%s] add_exclude(%.*s, %s%sclude)\n",
383 + rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n",
384 who_am_i(), (int)pat_len, cp, listp->debug_type,
385 + mflags & MATCHFLG_MERGE_FILE ? "FILE " : "",
386 mflags & MATCHFLG_INCLUDE ? "in" : "ex");
389 @@ -343,6 +573,11 @@ void add_exclude_file(struct exclude_lis
394 + rprintf(FINFO, "[%s] add_exclude_file(%s)\n",
395 + who_am_i(), fname);
400 int ch, overflow = 0;
401 @@ -402,7 +637,11 @@ void send_exclude_list(int f)
402 if (ent->match_flags & MATCHFLG_INCLUDE) {
404 write_buf(f, "+ ", 2);
405 - } else if ((*p == '-' || *p == '+') && p[1] == ' ') {
406 + } else if (ent->match_flags & MATCHFLG_MERGE_FILE) {
407 + write_int(f, l + 2);
408 + write_buf(f, ". ", 2);
409 + } else if ((*p == '-' || *p == '+' || *p == '.')
412 write_buf(f, "- ", 2);
414 @@ -443,6 +682,7 @@ void add_cvs_excludes(void)
415 char fname[MAXPATHLEN];
418 + add_exclude(&exclude_list, ". .cvsignore", 0);
419 add_exclude(&exclude_list, default_cvsignore,
420 XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
422 --- flist.c 21 May 2004 23:22:14 -0000 1.225
423 +++ flist.c 24 May 2004 00:44:00 -0000
424 @@ -39,8 +39,6 @@ extern int module_id;
425 extern int ignore_errors;
426 extern int numeric_ids;
428 -extern int cvs_exclude;
431 extern char curr_dir[MAXPATHLEN];
432 extern char *files_from;
433 @@ -65,7 +63,6 @@ extern int write_batch;
435 extern struct exclude_list_struct exclude_list;
436 extern struct exclude_list_struct server_exclude_list;
437 -extern struct exclude_list_struct local_exclude_list;
441 @@ -210,8 +207,6 @@ int link_stat(const char *path, STRUCT_S
443 static int check_exclude_file(char *fname, int is_dir, int exclude_level)
447 #if 0 /* This currently never happens, so avoid a useless compare. */
448 if (exclude_level == NO_EXCLUDES)
450 @@ -233,10 +228,7 @@ static int check_exclude_file(char *fnam
451 if (exclude_level != ALL_EXCLUDES)
453 if (exclude_list.head
454 - && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0)
456 - if (local_exclude_list.head
457 - && check_exclude(&local_exclude_list, fname, is_dir) < 0)
458 + && check_exclude(&exclude_list, fname, is_dir) < 0)
462 @@ -942,15 +934,7 @@ void send_file_name(int f, struct file_l
464 if (recursive && S_ISDIR(file->mode)
465 && !(file->flags & FLAG_MOUNT_POINT)) {
466 - struct exclude_list_struct last_list = local_exclude_list;
467 - local_exclude_list.head = local_exclude_list.tail = NULL;
468 send_directory(f, flist, f_name_to(file, fbuf));
470 - rprintf(FINFO, "[%s] popping %sexclude list\n",
471 - who_am_i(), local_exclude_list.debug_type);
473 - clear_exclude_list(&local_exclude_list);
474 - local_exclude_list = last_list;
478 @@ -961,6 +945,7 @@ static void send_directory(int f, struct
480 char fname[MAXPATHLEN];
482 + void *save_excludes;
486 @@ -984,18 +969,7 @@ static void send_directory(int f, struct
491 - if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset)
492 - < MAXPATHLEN - offset) {
493 - add_exclude_file(&local_exclude_list, fname,
494 - XFLG_WORD_SPLIT | XFLG_WORDS_ONLY);
496 - io_error |= IOERR_GENERAL;
498 - "cannot cvs-exclude in long-named directory %s\n",
499 - full_fname(fname));
502 + save_excludes = push_local_excludes(fname, offset);
504 for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
505 char *dname = d_name(di);
506 @@ -1016,6 +990,8 @@ static void send_directory(int f, struct
507 rsyserr(FERROR, errno, "readdir(%s)", dir);
510 + pop_local_excludes(save_excludes);
515 --- rsync.h 16 May 2004 07:28:24 -0000 1.204
516 +++ rsync.h 24 May 2004 00:44:02 -0000
517 @@ -496,11 +496,16 @@ struct map_struct {
518 #define MATCHFLG_INCLUDE (1<<4) /* this is an include, not an exclude */
519 #define MATCHFLG_DIRECTORY (1<<5) /* this matches only directories */
520 #define MATCHFLG_CLEAR_LIST (1<<6) /* this item is the "!" token */
521 +#define MATCHFLG_MERGE_FILE (1<<7) /* specifies a file to merge */
522 +#define MATCHFLG_CVSIGNORE (1<<8) /* parse this as a .cvsignore file */
523 struct exclude_struct {
524 struct exclude_struct *next;
526 unsigned int match_flags;
530 + struct exclude_list_struct *mergelist;
534 struct exclude_list_struct {
535 --- rsync.yo 21 May 2004 09:44:32 -0000 1.170
536 +++ rsync.yo 24 May 2004 00:44:02 -0000
537 @@ -1090,6 +1090,74 @@ itemize(
538 it would be excluded by the "*")
541 +manpagesection(MERGING EXCLUDE FILES)
543 +You can merge whole files into an exclude file using a rule that starts
544 +with a ". " (a dot followed by a space) and has a filename in place of the
545 +pattern. There are two types of merge rules, single-instance and
549 + it() If the filename has no slashes in it, it is a per-directory merge;
550 + rsync scans every directory that it traverses for the named file, merging
551 + its contents (when it exists) at the start of this per-directory sub-list
552 + (subdirectories inherit the contents of their parent directories by
553 + default, and each subdirectory's rules have precedence over the parent
554 + directory's rules). When a per-directory merge file adds another
555 + per-directory merge file, it is only active in the subtree of the current
558 + it() If a filename has a slash in it, it is a single-instance merge; the
559 + named file's contents will be merged into the current exclude file,
560 + replacing the merge rule. Thus, you should use the name ./foo instead of
561 + foo if you don't want to scan for "foo" in all the subdirectories of the
565 +Note also that you can eliminate all the inherited rules for the current
566 +per-directory ruleset by putting the list-clearing token (!) in the file.
567 +This only clears the rules for the current per-directory sub-list (up
568 +through the ! token) and only for the current directory and its
571 +Here's an example exclude file (which you'd specify via the normal
572 +--exclude-from option):
575 + . /home/user/.global_excludes
582 +This will merge the contents of the /home/user/.global_excludes file at the
583 +start of the list and also turns the ".excl" filename into a per-directory
584 +exclude file whose local contents will be merged into the list in place of
587 +Additionally, you can affect where the --cvs-exclude (-C) option's
588 +inclusion of a per-directory .cvsignore file gets placed into your rules by
589 +adding an explicit merge rule for ".cvsignore". For instance, specifying
593 + rsync -avC --exclude='. .cvsignore' --exclude-from=foo a/ b
596 +will merge all the per-directory .cvsignore rules at the start of your list
597 +rather than at the end. This allows their dir-specific rules to supersede
598 +your rules instead of being subservient to them. (The global rules taken
599 +from the $HOME/.cvsignore file and from $CVSIGNORE are not affected by
602 +Note also that the parsing of any merge-file named ".cvsignore" is always
603 +done in a CVS-compatible manner, even if -C wasn't specified (i.e. the
604 +rules are always exclude rules (even when specified by an include option);
605 +they are split on whitespace; no special prefixes, list-clearing tokens, or
606 +comment characters are honored; and, for per-directory files,
607 +subdirectories don't inherit the parent directory's rules).
609 manpagesection(BATCH MODE)
611 bf(Note:) Batch mode should be considered experimental in this version
612 --- testsuite/exclude.test 24 May 2004 00:16:07 -0000 1.8
613 +++ testsuite/exclude.test 24 May 2004 00:44:02 -0000
614 @@ -23,19 +23,47 @@ export HOME CVSIGNORE
615 makepath "$fromdir/foo/down/to/you"
616 makepath "$fromdir/bar/down/to/foo/too"
617 makepath "$fromdir/mid/for/foo/and/that/is/who"
618 +cat >"$fromdir/.excl" <<EOF
624 echo kept >"$fromdir/foo/file1"
625 echo removed >"$fromdir/foo/file2"
626 echo cvsout >"$fromdir/foo/file2.old"
627 +cat >"$fromdir/foo/.excl" <<EOF
631 +cat >"$fromdir/bar/.excl" <<EOF
636 echo cvsout >"$fromdir/bar/down/to/home-cvs-exclude"
637 +cat >"$fromdir/bar/down/to/.excl2" <<EOF
640 echo keeper >"$fromdir/bar/down/to/foo/file1"
641 echo cvsout >"$fromdir/bar/down/to/foo/file1.bak"
642 echo gone >"$fromdir/bar/down/to/foo/file3"
643 echo lost >"$fromdir/bar/down/to/foo/file4"
644 echo cvsout >"$fromdir/bar/down/to/foo/file4.junk"
645 echo smashed >"$fromdir/bar/down/to/foo/to"
646 +cat >"$fromdir/bar/down/to/foo/.excl2" <<EOF
649 +# This one should be ineffectual
650 +cat >"$fromdir/mid/.excl2" <<EOF
653 echo cvsout >"$fromdir/mid/one-in-one-out"
654 echo one-in-one-out >"$fromdir/mid/.cvsignore"
655 echo cvsin >"$fromdir/mid/one-for-all"
656 +cat >"$fromdir/mid/.excl" <<EOF
659 echo cvsin >"$fromdir/mid/for/one-in-one-out"
660 echo expunged >"$fromdir/mid/for/foo/extra"
661 echo retained >"$fromdir/mid/for/foo/keep"
662 @@ -100,5 +128,24 @@ $RSYNC -av --existing --include='*/' --e
663 checkit "$RSYNC -avvC --exclude-from=\"$excl\" \
664 --delete-excluded \"$fromdir/\" \"$todir/\"" "$chkdir" "$todir"
666 +# Modify the chk dir for our merge-exclude test and then tweak the dir times.
669 +rm "$chkdir"/foo/file1
670 +rm "$chkdir"/bar/.excl
671 +rm "$chkdir"/bar/down/to/.excl2
672 +rm "$chkdir"/bar/down/to/foo/.excl2
673 +rm "$chkdir"/mid/.excl
674 +cp -p "$fromdir"/bar/down/to/foo/*.junk "$chkdir"/bar/down/to/foo
675 +cp -p "$fromdir"/bar/down/to/foo/to "$chkdir"/bar/down/to/foo
677 +$RSYNC -av --existing --include='*/' --exclude='*' "$fromdir/" "$chkdir/"
679 +# Now, test if rsync excludes the same files, this time with a merge-exclude
682 +checkit "$RSYNC -avv --exclude='. .excl' --exclude-from=\"$excl\" \
683 + --delete-excluded \"$fromdir/\" \"$todir/\"" "$chkdir" "$todir"
685 # The script would have aborted on error, so getting here means we've won.