After applying this patch and running configure, you MUST run this command before "make": make proto This patch adds the ability to merge rules into your excludes/includes using a ". FILE" idiom. If you specify a name with a preceding -p option, that filename will be looked for in every subdirectory that rsync visits, and the rules found in that subdirectory's file will affect that dir and its subdirs. For example: rsync -av --exclude='. -p .excl' from/ to The above will look for a file named ".excl" in every directory of the hierarchy that rsync visits, and it will exclude (by default) names based on the rules found therein. If one of the .excl files contains this: + *.c . -p .excl2 . .excl3 *.o /foobar Then the file ".excl2" will also be read in from the current dir and all its subdirs (due to the -p option). The file ".excl3" would just be read in from the current dir. The exclusion of "foobar" will only happen in that .excl file's directory because the rule is anchored (so that's how you can make rules local instead of inherited). ..wayne.. --- orig/clientserver.c 2004-08-02 02:29:16 +++ clientserver.c 2004-08-10 15:44:15 @@ -48,12 +48,14 @@ extern int no_detach; extern int default_af_hint; extern char *bind_address; extern struct exclude_list_struct server_exclude_list; -extern char *exclude_path_prefix; extern char *config_file; extern char *files_from; char *auth_user; +/* Length of lp_path() string when in daemon mode & not chrooted, else 0. */ +unsigned int module_dirlen = 0; + /** * Run a client connected to an rsyncd. The alternative to this * function for remote-shell connections is do_cmd(). @@ -300,26 +302,28 @@ static int rsync_module(int f_in, int f_ /* TODO: Perhaps take a list of gids, and make them into the * supplementary groups. */ - exclude_path_prefix = use_chroot? "" : lp_path(i); - if (*exclude_path_prefix == '/' && !exclude_path_prefix[1]) - exclude_path_prefix = ""; + if (use_chroot) { + module_dirlen = 0; + set_excludes_dir("/", 1); + } else { + module_dirlen = strlen(lp_path(i)); + set_excludes_dir(lp_path(i), module_dirlen); + } p = lp_include_from(i); add_exclude_file(&server_exclude_list, p, - XFLG_FATAL_ERRORS | XFLG_DEF_INCLUDE); + XFLG_FATAL_ERRORS | XFLG_DEF_INCLUDE | XFLG_ABS_PATH); p = lp_include(i); add_exclude(&server_exclude_list, p, - XFLG_WORD_SPLIT | XFLG_DEF_INCLUDE); + XFLG_WORD_SPLIT | XFLG_DEF_INCLUDE | XFLG_ABS_PATH); p = lp_exclude_from(i); add_exclude_file(&server_exclude_list, p, - XFLG_FATAL_ERRORS); + XFLG_FATAL_ERRORS | XFLG_ABS_PATH); p = lp_exclude(i); - add_exclude(&server_exclude_list, p, XFLG_WORD_SPLIT); - - exclude_path_prefix = NULL; + add_exclude(&server_exclude_list, p, XFLG_WORD_SPLIT | XFLG_ABS_PATH); log_init(); --- orig/exclude.c 2004-08-10 18:17:01 +++ exclude.c 2004-08-13 07:40:08 @@ -30,13 +30,69 @@ extern int verbose; extern int eol_nulls; extern int list_only; extern int recurse; +extern int io_error; +extern int sanitize_paths; extern char curr_dir[]; +extern unsigned int curr_dir_len; +extern unsigned int module_dirlen; struct exclude_list_struct exclude_list = { 0, 0, "" }; -struct exclude_list_struct local_exclude_list = { 0, 0, "per-dir .cvsignore " }; struct exclude_list_struct server_exclude_list = { 0, 0, "server " }; -char *exclude_path_prefix = NULL; + +struct mergelist_save_struct { + struct exclude_list_struct *array; + int count; +}; + +/* The dirbuf is set by push_local_excludes() to the current subdirectory + * relative to curr_dir that is being processed. The path always has a + * trailing slash appended, and the variable dirbuf_len contains the length + * of this path prefix. The path is always absolute. */ +static char dirbuf[MAXPATHLEN+1]; +static unsigned int dirbuf_len = 0; +static int dirbuf_depth; + +/* This is True when we're scanning parent dirs for per-dir merge-files. */ +static BOOL parent_dirscan = False; + +/* This array contains a list of all the currently active per-dir merge + * files. This makes it easier to save the appropriate values when we + * "push" down into each subdirectory. */ +static struct exclude_struct **mergelist_parents; +static int mergelist_cnt = 0; +static int mergelist_size = 0; + +/* Each exclude_list_struct describes a singly-linked list by keeping track + * of both the head and tail pointers. The list is slightly unusual in that + * a parent-dir's content can be appended to the end of the local list in a + * special way: the last item in the local list has its "next" pointer set + * to point to the inherited list, but the local list's tail pointer points + * at the end of the local list. Thus, if the local list is empty, the head + * will be pointing at the inherited content but the tail will be NULL. To + * help you visualize this, here are the possible list arrangements: + * + * Completely Empty Local Content Only + * ================================== ==================================== + * head -> NULL head -> Local1 -> Local2 -> NULL + * tail -> NULL tail -------------^ + * + * Inherited Content Only Both Local and Inherited Content + * ================================== ==================================== + * head -> Parent1 -> Parent2 -> NULL head -> L1 -> L2 -> P1 -> P2 -> NULL + * tail -> NULL tail ---------^ + * + * This means that anyone wanting to traverse the whole list to USE it just + * needs to start at the head and use the "next" pointers until it goes + * NULL. To add new local content, we insert the item after the tail item + * and update the tail (obviously, if "tail" was NULL, we insert it at the + * head). To clear the local list, WE MUST NOT FREE THE INHERITED CONTENT + * because it is shared between the current list and our parent list(s). + * The easiest way to handle this is to simply truncate the list after the + * tail item and then free the local list from the head. When inheriting + * the list for a new local dir, we just save off the exclude_list_struct + * values (so we can pop back to them later) and set the tail to NULL. + */ /** Build an exclude structure given an exclude pattern. */ static void make_exclude(struct exclude_list_struct *listp, const char *pat, @@ -46,23 +102,50 @@ static void make_exclude(struct exclude_ const char *cp; unsigned int ex_len; + if (verbose > 2) { + rprintf(FINFO, "[%s] add_exclude(%.*s, %s%s%sclude)\n", + who_am_i(), (int)pat_len, pat, listp->debug_type, + mflags & MATCHFLG_MERGE_FILE ? "FILE " : "", + mflags & MATCHFLG_INCLUDE ? "in" : "ex"); + } + + if (mflags & MATCHFLG_MERGE_FILE) { + int i; + /* If the local include file was already mentioned, don't + * add it again. */ + for (i = 0; i < mergelist_cnt; i++) { + struct exclude_struct *ex = mergelist_parents[i]; + if (strlen(ex->pattern) == pat_len + && memcmp(ex->pattern, pat, pat_len) == 0) + return; + } + if ((pat_len == 10 || (pat_len > 10 && pat[pat_len-11] == '/')) + && strncmp(pat+pat_len-10, ".cvsignore", 10) == 0) { + mflags |= MATCHFLG_CVSIGNORE; + mflags &= ~MATCHFLG_INCLUDE; + } else + mflags &= ~MATCHFLG_CVSIGNORE; + } + ret = new(struct exclude_struct); if (!ret) out_of_memory("make_exclude"); memset(ret, 0, sizeof ret[0]); - if (exclude_path_prefix) - mflags |= MATCHFLG_ABS_PATH; - if (exclude_path_prefix && *pat == '/') - ex_len = strlen(exclude_path_prefix); - else + if (mflags & MATCHFLG_ABS_PATH) { + if (*pat != '/') { + mflags &= ~MATCHFLG_ABS_PATH; + ex_len = 0; + } else + ex_len = dirbuf_len - module_dirlen - 1; + } else ex_len = 0; ret->pattern = new_array(char, ex_len + pat_len + 1); if (!ret->pattern) out_of_memory("make_exclude"); if (ex_len) - memcpy(ret->pattern, exclude_path_prefix, ex_len); + memcpy(ret->pattern, dirbuf + module_dirlen, ex_len); strlcpy(ret->pattern + ex_len, pat, pat_len + 1); pat_len += ex_len; @@ -81,14 +164,40 @@ static void make_exclude(struct exclude_ mflags |= MATCHFLG_DIRECTORY; } - for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++) - ret->slash_cnt++; + if (mflags & MATCHFLG_MERGE_FILE) { + struct exclude_list_struct *lp + = new_array(struct exclude_list_struct, 1); + if (!lp) + out_of_memory("make_exclude"); + lp->head = lp->tail = NULL; + if ((cp = strrchr(ret->pattern, '/')) != NULL) + cp++; + else + cp = ret->pattern; + if (asprintf(&lp->debug_type, "per-dir %s ", cp) < 0) + out_of_memory("make_exclude"); + ret->u.mergelist = lp; + if (mergelist_cnt == mergelist_size) { + mergelist_size += 5; + mergelist_parents = realloc_array(mergelist_parents, + struct exclude_struct *, + mergelist_size); + if (!mergelist_parents) + out_of_memory("make_exclude"); + } + mergelist_parents[mergelist_cnt++] = ret; + } else { + for (cp = ret->pattern; (cp = strchr(cp, '/')) != NULL; cp++) + ret->u.slash_cnt++; + } ret->match_flags = mflags; - if (!listp->tail) + if (!listp->tail) { + ret->next = listp->head; listp->head = listp->tail = ret; - else { + } else { + ret->next = listp->tail->next; listp->tail->next = ret; listp->tail = ret; } @@ -96,22 +205,267 @@ static void make_exclude(struct exclude_ static void free_exclude(struct exclude_struct *ex) { + if (ex->match_flags & MATCHFLG_MERGE_FILE) { + free(ex->u.mergelist->debug_type); + free(ex->u.mergelist); + } free(ex->pattern); free(ex); } -void clear_exclude_list(struct exclude_list_struct *listp) +static void clear_exclude_list(struct exclude_list_struct *listp) { - struct exclude_struct *ent, *next; - - for (ent = listp->head; ent; ent = next) { - next = ent->next; - free_exclude(ent); + if (listp->tail) { + struct exclude_struct *ent, *next; + /* Truncate any inherited items from the local list. */ + listp->tail->next = NULL; + /* Now free everything that is left. */ + for (ent = listp->head; ent; ent = next) { + next = ent->next; + free_exclude(ent); + } } listp->head = listp->tail = NULL; } +/* This returns an expanded (absolute) filename for the merge-file name if + * the name has any slashes in it OR if the parent_dirscan var is True; + * otherwise it returns the original merge_file name. If the len_ptr value + * is non-NULL the merge_file name is limited by the referenced length + * value and will be updated with the length of the resulting name. We + * always return a name that is null terminated, even if the merge_file + * name was not. */ +static char *parse_merge_name(const char *merge_file, unsigned int *len_ptr, + unsigned int prefix_skip) +{ + static char buf[MAXPATHLEN]; + char *fn, tmpbuf[MAXPATHLEN]; + unsigned int fn_len; + + if (!parent_dirscan && *merge_file != '/') { + /* Return the name unchanged it doesn't have any slashes. */ + if (len_ptr) { + const char *p = merge_file + *len_ptr; + while (--p > merge_file && *p != '/') {} + if (p == merge_file) { + strlcpy(buf, merge_file, *len_ptr + 1); + return buf; + } + } else if (strchr(merge_file, '/') == NULL) + return (char *)merge_file; + } + + fn = *merge_file == '/' ? buf : tmpbuf; + if (sanitize_paths) { + const char *r = prefix_skip ? "/" : NULL; + /* null-terminate the name if it isn't already */ + if (len_ptr && merge_file[*len_ptr]) { + char *to = fn == buf ? tmpbuf : buf; + strlcpy(to, merge_file, *len_ptr + 1); + merge_file = to; + } + if (!sanitize_path(fn, merge_file, r, dirbuf_depth)) { + rprintf(FERROR, "merge-file name overflows: %s\n", + merge_file); + return NULL; + } + } else { + strlcpy(fn, merge_file, len_ptr ? *len_ptr + 1 : MAXPATHLEN); + clean_fname(fn, 1); + } + + fn_len = strlen(fn); + if (fn == buf) + goto done; + + if (dirbuf_len + fn_len >= MAXPATHLEN) { + rprintf(FERROR, "merge-file name overflows: %s\n", fn); + return NULL; + } + memcpy(buf, dirbuf + prefix_skip, dirbuf_len - prefix_skip); + memcpy(buf + dirbuf_len - prefix_skip, fn, fn_len + 1); + fn_len = clean_fname(buf, 1); + + done: + if (len_ptr) + *len_ptr = fn_len; + return buf; +} + +/* Sets the dirbuf and dirbuf_len values. */ +void set_excludes_dir(const char *dir, unsigned int dirlen) +{ + unsigned int len; + if (*dir != '/') { + memcpy(dirbuf, curr_dir, curr_dir_len); + dirbuf[curr_dir_len] = '/'; + len = curr_dir_len + 1; + if (len + dirlen >= MAXPATHLEN) + dirlen = 0; + } else + len = 0; + memcpy(dirbuf + len, dir, dirlen); + dirbuf[dirlen + len] = '\0'; + dirbuf_len = clean_fname(dirbuf, 1); + if (dirbuf_len > 1 && dirbuf[dirbuf_len-1] == '.' + && dirbuf[dirbuf_len-2] == '/') + dirbuf_len -= 2; + dirbuf[dirbuf_len++] = '/'; + dirbuf[dirbuf_len] = '\0'; + if (sanitize_paths) + dirbuf_depth = count_dir_elements(dirbuf + module_dirlen); +} + +/* This routine takes a per-dir merge-file entry and finishes its setup. + * If the name has a path portion then we check to see if it refers to a + * parent directory of the first transfer dir. If it does, we scan all the + * dirs from that point through the parent dir of the transfer dir looking + * for the per-dir merge-file in each one. */ +static BOOL setup_merge_file(struct exclude_struct *ex, + struct exclude_list_struct *lp, int flags) +{ + char buf[MAXPATHLEN]; + char *x, *y, *pat = ex->pattern; + unsigned int len; + + if (!(x = parse_merge_name(pat, NULL, 0)) || *x != '/') + return 0; + + y = strrchr(x, '/'); + *y = '\0'; + ex->pattern = strdup(y+1); + if (!*x) + x = "/"; + if (*x == '/') + strlcpy(buf, x, MAXPATHLEN); + else + pathjoin(buf, MAXPATHLEN, dirbuf, x); + + len = clean_fname(buf, 1); + if (len != 1 && len < MAXPATHLEN-1) { + buf[len++] = '/'; + buf[len] = '\0'; + } + /* This ensures that the specified dir is a parent of the transfer. */ + for (x = buf, y = dirbuf; *x && *x == *y; x++, y++) {} + if (*x) + y += strlen(y); /* nope -- skip the scan */ + + parent_dirscan = True; + while (*y) { + char save[MAXPATHLEN]; + strlcpy(save, y, MAXPATHLEN); + *y = '\0'; + dirbuf_len = y - dirbuf; + strlcpy(x, ex->pattern, MAXPATHLEN - (x - buf)); + add_exclude_file(lp, buf, flags | XFLG_ABS_PATH); + if (ex->match_flags & MATCHFLG_CVSIGNORE) + lp->head = NULL; /* CVS doesn't inherit rules. */ + lp->tail = NULL; + strlcpy(y, save, MAXPATHLEN); + while ((*x++ = *y++) != '/') {} + } + parent_dirscan = False; + free(pat); + return 1; +} + +/* Each time rsync changes to a new directory it call this function to + * handle all the per-dir merge-files. The "dir" value is the current path + * relative to curr_dir (which might not be null-terminated). We copy it + * into dirbuf so that we can easily append a file name on the end. */ +void *push_local_excludes(const char *dir, unsigned int dirlen) +{ + struct mergelist_save_struct *push; + struct exclude_list_struct *ap; + int i; + + set_excludes_dir(dir, dirlen); + + if (!(push = new_array(struct mergelist_save_struct, 1))) + out_of_memory("push_local_excludes"); + + push->count = mergelist_cnt; + push->array = new_array(struct exclude_list_struct, mergelist_cnt); + if (!push->array) + out_of_memory("push_local_excludes"); + + for (i = 0, ap = push->array; i < mergelist_cnt; i++) { + memcpy(ap++, mergelist_parents[i]->u.mergelist, + sizeof (struct exclude_list_struct)); + } + + /* Note: add_exclude_file() might increase mergelist_cnt, so keep + * this loop separate from the above loop. */ + for (i = 0; i < mergelist_cnt; i++) { + struct exclude_struct *ex = mergelist_parents[i]; + struct exclude_list_struct *lp = ex->u.mergelist; + int flags; + + if (verbose > 2) { + rprintf(FINFO, "[%s] pushing %sexclude list\n", + who_am_i(), lp->debug_type); + } + + if (ex->match_flags & MATCHFLG_CVSIGNORE) { + lp->head = NULL; /* CVS doesn't inherit rules. */ + flags = XFLG_WORD_SPLIT | XFLG_WORDS_ONLY; + } else { + flags = ex->match_flags & MATCHFLG_INCLUDE + ? XFLG_DEF_INCLUDE : 0; + } + lp->tail = NULL; /* Switch any local rules to inherited. */ + + if (ex->match_flags & MATCHFLG_FINISH_SETUP) { + ex->match_flags &= ~MATCHFLG_FINISH_SETUP; + if (setup_merge_file(ex, lp, flags)) + set_excludes_dir(dir, dirlen); + } + + if (strlcpy(dirbuf + dirbuf_len, ex->pattern, + MAXPATHLEN - dirbuf_len) < MAXPATHLEN - dirbuf_len) + add_exclude_file(lp, dirbuf, flags | XFLG_ABS_PATH); + else { + io_error |= IOERR_GENERAL; + rprintf(FINFO, + "cannot add local excludes in long-named directory %s\n", + full_fname(dirbuf)); + } + dirbuf[dirbuf_len] = '\0'; + } + + return (void*)push; +} + +void pop_local_excludes(void *mem) +{ + struct mergelist_save_struct *pop = (struct mergelist_save_struct*)mem; + struct exclude_list_struct *ap; + int i; + + for (i = mergelist_cnt; i-- > 0; ) { + struct exclude_struct *ex = mergelist_parents[i]; + struct exclude_list_struct *lp = ex->u.mergelist; + + if (verbose > 2) { + rprintf(FINFO, "[%s] popping %sexclude list\n", + who_am_i(), lp->debug_type); + } + + clear_exclude_list(lp); + } + + mergelist_cnt = pop->count; + for (i = 0, ap = pop->array; i < mergelist_cnt; i++) { + memcpy(mergelist_parents[i]->u.mergelist, ap++, + sizeof (struct exclude_list_struct)); + } + + free(pop->array); + free(pop); +} + static int check_one_exclude(char *name, struct exclude_struct *ex, int name_is_dir) { @@ -125,13 +479,14 @@ static int check_one_exclude(char *name, /* If the pattern does not have any slashes AND it does not have * a "**" (which could match a slash), then we just match the * name portion of the path. */ - if (!ex->slash_cnt && !(ex->match_flags & MATCHFLG_WILD2)) { + if (!ex->u.slash_cnt && !(ex->match_flags & MATCHFLG_WILD2)) { if ((p = strrchr(name,'/')) != NULL) name = p+1; } else if (ex->match_flags & MATCHFLG_ABS_PATH && *name != '/' - && curr_dir[1]) { - pathjoin(full_name, sizeof full_name, curr_dir + 1, name); + && curr_dir_len > module_dirlen + 1) { + pathjoin(full_name, sizeof full_name, + curr_dir + module_dirlen + 1, name); name = full_name; } @@ -148,9 +503,9 @@ static int check_one_exclude(char *name, if (ex->match_flags & MATCHFLG_WILD) { /* A non-anchored match with an infix slash and no "**" * needs to match the last slash_cnt+1 name elements. */ - if (!match_start && ex->slash_cnt + if (!match_start && ex->u.slash_cnt && !(ex->match_flags & MATCHFLG_WILD2)) { - int cnt = ex->slash_cnt + 1; + int cnt = ex->u.slash_cnt + 1; for (p = name + strlen(name) - 1; p >= name; p--) { if (*p == '/' && !--cnt) break; @@ -221,6 +576,13 @@ int check_exclude(struct exclude_list_st struct exclude_struct *ent; for (ent = listp->head; ent; ent = ent->next) { + if (ent->match_flags & MATCHFLG_MERGE_FILE) { + int rc = check_exclude(ent->u.mergelist, name, + name_is_dir); + if (rc) + return rc; + continue; + } if (check_one_exclude(name, ent, name_is_dir)) { report_exclude_result(name, ent, name_is_dir, listp->debug_type); @@ -253,11 +615,36 @@ static const char *get_exclude_tok(const p = (const char *)s; } - /* Is this a '+' or '-' followed by a space (not whitespace)? */ + /* Check for a +/-/. followed by a space (not whitespace). */ if (!(xflags & XFLG_WORDS_ONLY) - && (*s == '-' || *s == '+') && s[1] == ' ') { + && (*s == '-' || *s == '+' || *s == '.') && s[1] == ' ') { if (*s == '+') mflags |= MATCHFLG_INCLUDE; + else if (*s == '.') { + mflags |= MATCHFLG_MERGE_FILE; + if (xflags & XFLG_DEF_INCLUDE) + mflags |= MATCHFLG_INCLUDE; + while (s[2] == '-') { + s += 2; + do { + switch (*++s) { + case 'p': + mflags |= MATCHFLG_PERDIR_MERGE + | MATCHFLG_FINISH_SETUP; + break; + case '-': + if (s[1] == ' ') + goto done; + default: + rprintf(FERROR, + "invalid merge options: %s\n", + p); + exit_cleanup(RERR_SYNTAX); + } + } while (s[1] != ' '); + } + } + done: s += 2; } else if (xflags & XFLG_DEF_INCLUDE) mflags |= MATCHFLG_INCLUDE; @@ -273,6 +660,8 @@ static const char *get_exclude_tok(const if (*p == '!' && len == 1 && !(xflags & XFLG_WORDS_ONLY)) mflags |= MATCHFLG_CLEAR_LIST; + if (xflags & XFLG_ABS_PATH) + mflags |= MATCHFLG_ABS_PATH; *len_ptr = len; *flag_ptr = mflags; @@ -284,7 +673,7 @@ void add_exclude(struct exclude_list_str int xflags) { unsigned int pat_len, mflags; - const char *cp; + const char *cp, *p; if (!pattern) return; @@ -292,9 +681,15 @@ void add_exclude(struct exclude_list_str cp = pattern; pat_len = 0; while (1) { + /* Remember that the returned string is NOT '\0' terminated! */ cp = get_exclude_tok(cp + pat_len, &pat_len, &mflags, xflags); if (!pat_len) break; + if (pat_len >= MAXPATHLEN) { + rprintf(FERROR, "discarding over-long exclude: %s\n", + cp); + continue; + } if (mflags & MATCHFLG_CLEAR_LIST) { if (verbose > 2) { @@ -306,13 +701,24 @@ void add_exclude(struct exclude_list_str continue; } - make_exclude(listp, cp, pat_len, mflags); - - if (verbose > 2) { - rprintf(FINFO, "[%s] add_exclude(%.*s, %s%sclude)\n", - who_am_i(), (int)pat_len, cp, listp->debug_type, - mflags & MATCHFLG_INCLUDE ? "in" : "ex"); + if (mflags & MATCHFLG_MERGE_FILE) { + unsigned int len = pat_len; + if (mflags & MATCHFLG_PERDIR_MERGE) { + if (parent_dirscan) { + if (!(p = parse_merge_name(cp, &len, module_dirlen))) + continue; + make_exclude(listp, p, len, mflags); + continue; + } + } else { + if (!(p = parse_merge_name(cp, &len, 0))) + continue; + add_exclude_file(listp, p, xflags | XFLG_FATAL_ERRORS); + continue; + } } + + make_exclude(listp, cp, pat_len, mflags); } } @@ -321,7 +727,7 @@ void add_exclude_file(struct exclude_lis int xflags) { FILE *fp; - char line[MAXPATHLEN+3]; /* Room for "x " prefix and trailing slash. */ + char line[MAXPATHLEN+7]; /* Room for prefix chars and trailing slash. */ char *eob = line + sizeof line - 1; int word_split = xflags & XFLG_WORD_SPLIT; @@ -342,6 +748,12 @@ void add_exclude_file(struct exclude_lis } return; } + dirbuf[dirbuf_len] = '\0'; + + if (verbose > 2) { + rprintf(FINFO, "[%s] add_exclude_file(%s,%d)\n", + who_am_i(), fname, xflags); + } while (1) { char *s = line; @@ -402,7 +814,21 @@ void send_exclude_list(int f) if (ent->match_flags & MATCHFLG_INCLUDE) { write_int(f, l + 2); write_buf(f, "+ ", 2); - } else if ((*p == '-' || *p == '+') && p[1] == ' ') { + } else if (ent->match_flags & MATCHFLG_MERGE_FILE) { + char buf[32], *op = buf; + *op++ = '.'; + *op++ = ' '; + if (ent->match_flags & MATCHFLG_PERDIR_MERGE) { + *op++ = '-'; + *op++ = 'p'; + if (*p == '-') + *op++ = '-'; + *op++ = ' '; + } + write_int(f, l + (op - buf)); + write_buf(f, buf, op - buf); + } else if ((*p == '-' || *p == '+' || *p == '.') + && p[1] == ' ') { write_int(f, l + 2); write_buf(f, "- ", 2); } else @@ -443,6 +869,7 @@ void add_cvs_excludes(void) char fname[MAXPATHLEN]; char *p; + add_exclude(&exclude_list, ". -p .cvsignore", 0); add_exclude(&exclude_list, default_cvsignore, XFLG_WORD_SPLIT | XFLG_WORDS_ONLY); --- orig/flist.c 2004-08-12 18:34:38 +++ flist.c 2004-08-12 18:59:28 @@ -39,10 +39,9 @@ extern int module_id; extern int ignore_errors; extern int numeric_ids; -extern int cvs_exclude; - extern int recurse; extern char curr_dir[MAXPATHLEN]; +extern unsigned int curr_dir_len; extern char *files_from; extern int filesfrom_fd; @@ -66,7 +65,6 @@ extern int list_only; extern struct exclude_list_struct exclude_list; extern struct exclude_list_struct server_exclude_list; -extern struct exclude_list_struct local_exclude_list; int io_error; @@ -221,8 +219,6 @@ int link_stat(const char *path, STRUCT_S */ static int check_exclude_file(char *fname, int is_dir, int exclude_level) { - int rc; - #if 0 /* This currently never happens, so avoid a useless compare. */ if (exclude_level == NO_EXCLUDES) return 0; @@ -244,10 +240,7 @@ static int check_exclude_file(char *fnam if (exclude_level != ALL_EXCLUDES) return 0; if (exclude_list.head - && (rc = check_exclude(&exclude_list, fname, is_dir)) != 0) - return rc < 0; - if (local_exclude_list.head - && check_exclude(&local_exclude_list, fname, is_dir) < 0) + && check_exclude(&exclude_list, fname, is_dir) < 0) return 1; return 0; } @@ -956,15 +949,7 @@ void send_file_name(int f, struct file_l if (recursive && S_ISDIR(file->mode) && !(file->flags & FLAG_MOUNT_POINT)) { - struct exclude_list_struct last_list = local_exclude_list; - local_exclude_list.head = local_exclude_list.tail = NULL; send_directory(f, flist, f_name_to(file, fbuf)); - if (verbose > 2) { - rprintf(FINFO, "[%s] popping %sexclude list\n", - who_am_i(), local_exclude_list.debug_type); - } - clear_exclude_list(&local_exclude_list); - local_exclude_list = last_list; } } @@ -975,6 +960,7 @@ static void send_directory(int f, struct struct dirent *di; char fname[MAXPATHLEN]; unsigned int offset; + void *save_excludes; char *p; d = opendir(dir); @@ -998,18 +984,7 @@ static void send_directory(int f, struct offset++; } - if (cvs_exclude) { - if (strlcpy(p, ".cvsignore", MAXPATHLEN - offset) - < MAXPATHLEN - offset) { - add_exclude_file(&local_exclude_list, fname, - XFLG_WORD_SPLIT | XFLG_WORDS_ONLY); - } else { - io_error |= IOERR_GENERAL; - rprintf(FINFO, - "cannot cvs-exclude in long-named directory %s\n", - full_fname(fname)); - } - } + save_excludes = push_local_excludes(fname, offset); for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) { char *dname = d_name(di); @@ -1030,6 +1005,8 @@ static void send_directory(int f, struct rsyserr(FERROR, errno, "readdir(%s)", dir); } + pop_local_excludes(save_excludes); + closedir(d); } @@ -1049,6 +1026,7 @@ struct file_list *send_file_list(int f, char *p, *dir, olddir[sizeof curr_dir]; char lastpath[MAXPATHLEN] = ""; struct file_list *flist; + BOOL need_first_push = True; int64 start_write; int use_ff_fd = 0; @@ -1069,6 +1047,10 @@ struct file_list *send_file_list(int f, exit_cleanup(RERR_FILESELECT); } use_ff_fd = 1; + if (curr_dir_len < MAXPATHLEN - 1) { + push_local_excludes(curr_dir, curr_dir_len); + need_first_push = False; + } } } @@ -1099,6 +1081,15 @@ struct file_list *send_file_list(int f, } } + if (need_first_push) { + if ((p = strrchr(fname, '/')) != NULL) { + if (*++p && strcmp(p, ".") != 0) + push_local_excludes(fname, p - fname); + } else if (strcmp(fname, ".") != 0) + push_local_excludes(fname, 0); + need_first_push = False; + } + if (link_stat(fname, &st, keep_dirlinks) != 0) { if (f != -1) { io_error |= IOERR_GENERAL; --- orig/options.c 2004-08-12 18:34:38 +++ options.c 2004-08-12 18:59:28 @@ -287,6 +287,7 @@ void usage(enum logcode F) rprintf(F," --include=PATTERN don't exclude files matching PATTERN\n"); rprintf(F," --include-from=FILE don't exclude patterns listed in FILE\n"); rprintf(F," --files-from=FILE read FILE for list of source-file names\n"); + rprintf(F," -E same as --exclude='. -p /.rsync-excludes'\n"); rprintf(F," -0, --from0 all *-from file lists are delimited by nulls\n"); rprintf(F," --version print version number\n"); rprintf(F," --daemon run as an rsync daemon\n"); @@ -389,6 +390,7 @@ static struct poptOption long_options[] {"ignore-errors", 0, POPT_ARG_NONE, &ignore_errors, 0, 0, 0 }, {"blocking-io", 0, POPT_ARG_VAL, &blocking_io, 1, 0, 0 }, {"no-blocking-io", 0, POPT_ARG_VAL, &blocking_io, 0, 0, 0 }, + {0, 'E', POPT_ARG_NONE, 0, 'E', 0, 0 }, {0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 }, {"config", 0, POPT_ARG_STRING, &config_file, 0, 0, 0 }, {"port", 0, POPT_ARG_INT, &rsync_port, 0, 0, 0 }, @@ -589,6 +591,11 @@ int parse_arguments(int *argc, const cha am_sender = 1; break; + case 'E': + add_exclude(&exclude_list, + ". -p /.rsync-excludes", 0); + break; + case 'P': do_progress = 1; keep_partial = 1; --- orig/rsync.h 2004-08-03 15:41:32 +++ rsync.h 2004-08-08 06:07:01 @@ -108,6 +108,7 @@ #define XFLG_DEF_INCLUDE (1<<1) #define XFLG_WORDS_ONLY (1<<2) #define XFLG_WORD_SPLIT (1<<3) +#define XFLG_ABS_PATH (1<<4) #define PERMS_REPORT (1<<0) #define PERMS_SKIP_MTIME (1<<1) @@ -499,11 +500,18 @@ struct map_struct { #define MATCHFLG_INCLUDE (1<<4) /* this is an include, not an exclude */ #define MATCHFLG_DIRECTORY (1<<5) /* this matches only directories */ #define MATCHFLG_CLEAR_LIST (1<<6) /* this item is the "!" token */ +#define MATCHFLG_MERGE_FILE (1<<7) /* specifies a file to merge */ +#define MATCHFLG_CVSIGNORE (1<<8) /* parse this as a .cvsignore file */ +#define MATCHFLG_PERDIR_MERGE (1<<9) /* merge-file is searched per-dir */ +#define MATCHFLG_FINISH_SETUP (1<<10)/* per-dir merge file needs setup */ struct exclude_struct { struct exclude_struct *next; char *pattern; unsigned int match_flags; - int slash_cnt; + union { + int slash_cnt; + struct exclude_list_struct *mergelist; + } u; }; struct exclude_list_struct { --- orig/rsync.yo 2004-08-13 07:18:59 +++ rsync.yo 2004-08-13 00:43:31 @@ -364,6 +364,7 @@ verb( --include=PATTERN don't exclude files matching PATTERN --include-from=FILE don't exclude patterns listed in FILE --files-from=FILE read FILE for list of source-file names + -E same as --exclude='. -p /.rsync-excludes' -0 --from0 all file lists are delimited by nulls --version print version number --daemon run as an rsync daemon @@ -1008,24 +1009,32 @@ The exclude and include patterns specifi selection of which files to transfer and which files to skip. Rsync builds an ordered list of include/exclude options as specified on -the command line. Rsync checks each file and directory -name against each exclude/include pattern in turn. The first matching +the command line. +It can also be told to check for include/exclude options in each +directory that rsync visits during the transfer (see the section on +MERGED EXCLUDE FILES for the details on these per-directory exclude +files). + +As the list of files/directories to transfer is built, rsync checks each +name against every exclude/include pattern in turn. The first matching pattern is acted on. If it is an exclude pattern, then that file is skipped. If it is an include pattern then that filename is not skipped. If no matching include/exclude pattern is found then the filename is not skipped. -The filenames matched against the exclude/include patterns are relative -to the "root of the transfer". If you think of the transfer as a -subtree of names that are being sent from sender to receiver, the root -is where the tree starts to be duplicated in the destination directory. -This root governs where patterns that start with a / match (see below). +The global include/exclude rules are anchored at the "root of the +transfer" (as opposed to per-directory rules, which are anchored at +the merge-file's directory). If you think of the transfer as a +subtree of names that are being sent from sender to receiver, the +transfer-root is where the tree starts to be duplicated in the +destination directory. This root governs where patterns that start +with a / match (as described in the list on pattern forms below). Because the matching is relative to the transfer-root, changing the trailing slash on a source path or changing your use of the --relative option affects the path you need to use in your matching (in addition to changing how much of the file tree is duplicated on the destination -system). The following examples demonstrate this. +host). The following examples demonstrate this. Let's say that we want to match two source files, one with an absolute path of "/home/me/foo/bar", and one with a path of "/home/you/bar/baz". @@ -1072,23 +1081,27 @@ because rsync did not descend through th hierarchy. Note also that the --include and --exclude options take one pattern -each. To add multiple patterns use the --include-from and ---exclude-from options or multiple --include and --exclude options. +each. To add multiple patterns use the --include-from and --exclude-from +options or multiple --include and --exclude options. -The patterns can take several forms. The rules are: +The include/exclude patterns can take several forms. The rules are: itemize( - it() if the pattern starts with a / then it is matched against the - start of the filename, otherwise it is matched against the end of - the filename. - This is the equivalent of a leading ^ in regular expressions. - Thus "/foo" would match a file called "foo" at the transfer-root - (see above for how this is different from the filesystem-root). - On the other hand, "foo" would match any file called "foo" + it() if the pattern starts with a / then it is anchored to a + particular spot in the hierarchy of files, otherwise it is matched + against the end of the pathname. This is similar to a leading ^ in + regular expressions. + Thus "/foo" would match a file called "foo" at either the "root of the + transfer" (for a global rule) or in the merge-file's directory (for a + per-directory rule). + An unqualified "foo" would match any file or directory named "foo" anywhere in the tree because the algorithm is applied recursively from + the top down; it behaves as if each path component gets a turn at being the - end of the file name. + end of the file name. Even the unanchored "sub/foo" would match at + any point in the hierarchy where a "foo" was found within a directory + named "sub". it() if the pattern ends with a / then it will only match a directory, not a file, link, or device. @@ -1101,22 +1114,31 @@ itemize( single asterisk pattern "*" will stop at slashes. it() if the pattern contains a / (not counting a trailing /) or a "**" - then it is matched against the full filename, including any leading - directory. If the pattern doesn't contain a / or a "**", then it is + then it is matched against the full pathname, including any leading + directories. If the pattern doesn't contain a / or a "**", then it is matched only against the final component of the filename. Again, remember that the algorithm is applied recursively so "full filename" can actually be any portion of a path below the starting directory. it() if the pattern starts with "+ " (a plus followed by a space) then it is always considered an include pattern, even if specified as - part of an exclude option. The prefix is discarded before matching. + part of an exclude option. (The prefix is discarded before matching.) it() if the pattern starts with "- " (a minus followed by a space) then it is always considered an exclude pattern, even if specified as - part of an include option. The prefix is discarded before matching. + part of an include option. (The prefix is discarded before matching.) + + it() if the pattern starts with ". " (a dot followed by a space) then its + pattern is taken to be a merge-file that is read in to supplement the + current rules. See the section on MERGED EXCLUDE FILES for more + information. it() if the pattern is a single exclamation mark ! then the current include/exclude list is reset, removing all previously defined patterns. + The "current" list is either the global list of rules (which are + specified via options) or a set of per-directory rules (which are + inherited in their own sub-list, so a subdirectory can use this to + clear out the parent's rules). ) The +/- rules are most useful in a list that was read from a file, allowing @@ -1163,8 +1185,160 @@ itemize( it() --include "*/" --include "*.c" --exclude "*" would include all directories and C source files it() --include "foo/" --include "foo/bar.c" --exclude "*" would include - only foo/bar.c (the foo/ directory must be explicitly included or - it would be excluded by the "*") + only the foo directory and foo/bar.c (the foo directory must be + explicitly included or it would be excluded by the "*") +) + +manpagesection(MERGED EXCLUDE FILES) + +You can merge whole files into an exclude file by specifying a rule that +starts with a ". " (a dot followed by a space) and putting a filename in +place of the pattern. There are two kinds of merged exclude files -- +single-instance and per-directory. The choice is made via an option +placed prior to the merge-file name: + +startdit() + +dit(bf(-p)) Make the file a per-directory merge-file. Rsync will scan +every directory that it traverses for the named file, merging its contents +when the file exists. These exclude files must exist on the sending side +because it is the sending side that is being scanned for available files +to send. The files may also need to be transferred to the receiving side +if you want them to affect what files don't get deleted (see PER-DIRECTORY +EXCLUDES AND DELETE below). + +dit(bf(--)) End the scanning of options. Useful if you want to specify a +filename that begins with a dash. + +enddit() + +Per-directory rules are inherited in all subdirectories of the directory +where the merge-file was found. Each subdirectory's rules are prefixed +to the inherited rules from the parent directories, which gives the +newest rules a higher priority than the inherited rules. The entire set +of per-dir rules is grouped together in the spot where the merge-file was +specified, so it is possible to override per-dir rules via a rule that +got specified earlier in the list of global rules. + +If you don't want a per-dir rule to be inherited, anchor it with a leading +slash. Anchored rules in a per-directory merge-file are relative to the +merge-file's directory, so a rule "/foo" would only exclude the file "foo" +in the directory where the per-dir exclude file was found. + +Here's an example exclude file which you'd specify via the normal +--exclude-from=FILE option: + +verb( + . /home/user/.global_excludes + *.gz + . -p .excl + + *.[ch] + *.o +) + +This will merge the contents of the /home/user/.global_excludes file at the +start of the list and also turns the ".excl" filename into a per-directory +exclude file. All the merged rules default to being exclude rules because +an exclude statement was used to specify them. Rules read in from the +.global_excludes file are anchored just like all other global rules. + +If a per-directory merge-file is specified with a path that is a parent +directory of the first transfer directory, rsync will scan all the parent +dirs from that starting point to the transfer directory for the indicated +per-directory file. For instance, the -E option is an abbreviation for +this command: + +verb( + --exclude='. -p /.rsync-excludes' +) + +That exclude tells rsync to scan for the file .rsync-excludes in all +directories from the root down through the source of the transfer. (For +an rsync daemon, the "root dir" is always the module's "path" setting.) + +Some examples of this pre-scanning for per-directory files: + +verb( + rsync -avE /src/path/ /dest/dir + rsync -av --exclude='. -p ../../.rsync-excludes' /src/path/ /dest/dir + rsync -av --exclude='. -p .rsync-excludes' /src/path/ /dest/dir +) + +The first two commands above will look for ".rsync-excludes" in "/" and +"/src" before the normal scan begins looking for the file in "/src/path" +and its subdirectories. The last command avoids the parent-dir scan +and only looks for the ".rsync-excludes" files in each directory that is +a part of the transfer. + +Finally, note that the parsing of any merge-file named ".cvsignore" is +always done in a CVS-compatible manner, even if -C wasn't specified. This +means that its rules are always excludes (even if an include option +specified the file), patterns are split on whitespace, the rules are never +inherited, and no special characters are honored (e.g. no comments, no "!", +etc.). + +Additionally, you can affect where the --cvs-exclude (-C) option's +inclusion of the per-directory .cvsignore file gets placed into your rules +by adding your own explicit per-directory merge rule for ".cvsignore". +Without this, rsync would add its this rule at the end of all your other +rules (giving it a lower priority than your command-line rules). For +example: + +verb( + rsync -avC --exclude='. -p .cvsignore' --exclude-from=foo a/ b +) + +The above will merge all the per-directory .cvsignore rules at the start of +your list rather than at the end. This allows their dir-specific rules to +supersede your rules instead of being subservient to them. (The global +rules taken from the $HOME/.cvsignore file and from $CVSIGNORE are not +repositioned by this.) + +manpagesection(PER-DIRECTORY EXCLUDES AND DELETE) + +Without a delete option, per-directory excludes are only relevant on the +sending side, so you can feel free to exclude the merge files themselves +without affecting the transfer: + +verb( + rsync -av --exclude='. -p .excl' --exclude=.excl host:src/dir /dest +) + +However, if you want to do a delete on the receiving side AND you want some +files to be excluded from being deleted, you'll need to be sure that the +receiving side knows what files to exclude. The easiest way is to include +the per-directory merge files in the transfer and use --delete-after +because this ensures that the receiving side gets all the same exclude +rules as the sending side before it tries to delete anything: + +verb( + rsync -avE --delete-after host:src/dir /dest +) + +However, if you the merge files are not a part of the transfer, you'll need +to either use a global exclude rule (i.e. specified on the command line), +or you'll need to maintain your own per-directory merge files on the +receiving side. An example of the first is this (assume that the remote +.ctrl files exclude themselves): + +verb( + rsync -av --exclude='. -p .ctrl' --exclude-from=/my/extra.rules + --delete host:src/dir /dest +) + +In the above example the extra.rules file can affect both sides of the +transfer, but the rules are subservient to the rules merged from the .ctrl +files because they were specified after the per-directory merge rule. + +In the final example, the remote side is excluding the .rsync-excludes +files from the transfer, but we want to use our own .rsync-excludes files +to control what gets deleted on the receiving side. To do this we must +specifically exclude the per-directory merge files (so that they don't get +deleted) and then put rules into the local files to control what else +should not get deleted. Like this: + +verb( + rsync -avE --exclude=.rsync-excludes --delete host:src/dir /dest ) manpagesection(BATCH MODE) --- orig/testsuite/exclude.test 2004-05-29 21:25:45 +++ testsuite/exclude.test 2004-08-08 06:35:15 @@ -23,19 +23,47 @@ export HOME CVSIGNORE makepath "$fromdir/foo/down/to/you" makepath "$fromdir/bar/down/to/foo/too" makepath "$fromdir/mid/for/foo/and/that/is/who" +cat >"$fromdir/.excl" <"$fromdir/foo/file1" echo removed >"$fromdir/foo/file2" echo cvsout >"$fromdir/foo/file2.old" +cat >"$fromdir/foo/.excl" <"$fromdir/bar/.excl" <"$fromdir/bar/down/to/home-cvs-exclude" +cat >"$fromdir/bar/down/to/.excl2" <"$fromdir/bar/down/to/foo/file1" echo cvsout >"$fromdir/bar/down/to/foo/file1.bak" echo gone >"$fromdir/bar/down/to/foo/file3" echo lost >"$fromdir/bar/down/to/foo/file4" echo cvsout >"$fromdir/bar/down/to/foo/file4.junk" echo smashed >"$fromdir/bar/down/to/foo/to" +cat >"$fromdir/bar/down/to/foo/.excl2" <"$fromdir/mid/.excl2" <"$fromdir/mid/one-in-one-out" echo one-in-one-out >"$fromdir/mid/.cvsignore" echo cvsin >"$fromdir/mid/one-for-all" +cat >"$fromdir/mid/.excl" <"$fromdir/mid/for/one-in-one-out" echo expunged >"$fromdir/mid/for/foo/extra" echo retained >"$fromdir/mid/for/foo/keep" @@ -100,5 +128,24 @@ $RSYNC -av --existing --include='*/' --e checkit "$RSYNC -avvC --exclude-from=\"$excl\" \ --delete-excluded \"$fromdir/\" \"$todir/\"" "$chkdir" "$todir" +# Modify the chk dir for our merge-exclude test and then tweak the dir times. + +rm "$chkdir"/.excl +rm "$chkdir"/foo/file1 +rm "$chkdir"/bar/.excl +rm "$chkdir"/bar/down/to/.excl2 +rm "$chkdir"/bar/down/to/foo/.excl2 +rm "$chkdir"/mid/.excl +cp -p "$fromdir"/bar/down/to/foo/*.junk "$chkdir"/bar/down/to/foo +cp -p "$fromdir"/bar/down/to/foo/to "$chkdir"/bar/down/to/foo + +$RSYNC -av --existing --include='*/' --exclude='*' "$fromdir/" "$chkdir/" + +# Now, test if rsync excludes the same files, this time with a merge-exclude +# file. + +checkit "$RSYNC -avv --exclude='. -p .excl' --exclude-from=\"$excl\" \ + --delete-excluded \"$fromdir/\" \"$todir/\"" "$chkdir" "$todir" + # The script would have aborted on error, so getting here means we've won. exit 0