From matt at mattmccutchen.net Sun Mar 8 21:07:57 2009 From: matt at mattmccutchen.net (Matt McCutchen) Date: Mon, 09 Mar 2009 00:07:57 -0400 Subject: [Rsync-patches] [PATCH] Give a meaningful error message when we fail to write to a batch file. In-Reply-To: <1233382457.24062.20.camel@mattlaptop2.local> References: <1232836803.10089.2.camel@mattlaptop2.local> <1233382457.24062.20.camel@mattlaptop2.local> Message-ID: <1236571677.3054.69.camel@mattlaptop2.local> Wayne, Bump. This patch has been in my repository for a month and you haven't expressed any opinion on it. -- Matt From matt at mattmccutchen.net Sun Mar 8 21:40:31 2009 From: matt at mattmccutchen.net (Matt McCutchen) Date: Mon, 09 Mar 2009 00:40:31 -0400 Subject: [Rsync-patches] [PATCH] Improvements related to missing source args. In-Reply-To: <1236564883.3054.68.camel@mattlaptop2.local> References: <1236564883.3054.68.camel@mattlaptop2.local> Message-ID: <1236573631.3054.74.camel@mattlaptop2.local> - Implement --ignore-missing-args. - Revise the man page. In particular, mention the perhaps surprising behavior that --delete-missing-args is governed by hide filters and not by protect filters. - In the absence of --*-missing-args, a missing source arg is a FERROR_XFER but doesn't need to be an IOERR_GENERAL. --- flist.c | 19 +++++++++++++------ generator.c | 6 +++--- options.c | 13 +++++++++---- rsync.yo | 23 +++++++++++++++-------- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/flist.c b/flist.c index 1c15d8c..a173ad1 100644 --- a/flist.c +++ b/flist.c @@ -51,7 +51,7 @@ extern int preserve_links; extern int preserve_hard_links; extern int preserve_devices; extern int preserve_specials; -extern int delete_missing_args; +extern int missing_args; extern int uid_ndx; extern int gid_ndx; extern int eol_nulls; @@ -2177,14 +2177,21 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) if (link_stat(fbuf, &st, copy_dirlinks || name_type != NORMAL_NAME) != 0 || (name_type != DOTDIR_NAME && is_daemon_excluded(fbuf, S_ISDIR(st.st_mode))) || (relative_paths && path_is_daemon_excluded(fbuf, 1))) { - if (errno == ENOENT && delete_missing_args) { - /* Rsync will treat a mode of 0 as deleted. */ - memset(&st, 0, sizeof st); - } else { - io_error |= IOERR_GENERAL; + if (errno != ENOENT || missing_args == 0) { + /* This is a transfer error, but inhibit deletion + * only if we might be omitting an existing file. */ + if (errno != ENOENT) + io_error |= IOERR_GENERAL; rsyserr(FERROR_XFER, errno, "link_stat %s failed", full_fname(fbuf)); continue; + } else if (missing_args == 1) { + /* Just ignore the arg. */ + continue; + } else /* (missing_args == 2) */ { + /* Send the arg as a "missing" entry with + * mode 0, which tells the generator to delete it. */ + memset(&st, 0, sizeof st); } } diff --git a/generator.c b/generator.c index 6dc4e7f..a1ef12b 100644 --- a/generator.c +++ b/generator.c @@ -50,7 +50,7 @@ extern int delete_mode; extern int delete_before; extern int delete_during; extern int delete_after; -extern int delete_missing_args; +extern int missing_args; extern int msgdone_cnt; extern int ignore_errors; extern int remove_source_files; @@ -1031,7 +1031,7 @@ static void list_file_entry(struct file_struct *f) F_SYMLINK(f)); } else #endif - if (delete_missing_args && !f->mode) { + if (missing_args == 2 && !f->mode) { rprintf(FINFO, "%-*s %s\n", colwidth + 31, "*missing", f_name(f, NULL)); @@ -1177,7 +1177,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, stat_errno = errno; } - if (delete_missing_args && file->mode == 0) { + if (missing_args == 2 && file->mode == 0) { if (statret == 0) delete_item(fname, sx.st.st_mode, del_opts); return; diff --git a/options.c b/options.c index 51697ad..d7c9e8e 100644 --- a/options.c +++ b/options.c @@ -69,7 +69,6 @@ int delete_during = 0; int delete_before = 0; int delete_after = 0; int delete_excluded = 0; -int delete_missing_args = 0; int remove_source_files = 0; int one_file_system = 0; int protocol_version = PROTOCOL_VERSION; @@ -82,6 +81,7 @@ int am_sender = 0; int am_starting_up = 1; int relative_paths = -1; int implied_dirs = 1; +int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */ int numeric_ids = 0; int msgs2stderr = 0; int allow_8bit_chars = 0; @@ -883,6 +883,8 @@ static struct poptOption long_options[] = { {"no-implied-dirs", 0, POPT_ARG_VAL, &implied_dirs, 0, 0, 0 }, {"i-d", 0, POPT_ARG_VAL, &implied_dirs, 1, 0, 0 }, {"no-i-d", 0, POPT_ARG_VAL, &implied_dirs, 0, 0, 0 }, + {"ignore-missing-args",0,POPT_ARG_VAL, &missing_args, 1, 0, 0 }, + {"delete-missing-args",0,POPT_ARG_VAL, &missing_args, 2, 0, 0 }, {"chmod", 0, POPT_ARG_STRING, 0, OPT_CHMOD, 0, 0 }, {"ignore-times", 'I', POPT_ARG_NONE, &ignore_times, 0, 0, 0 }, {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 }, @@ -910,7 +912,6 @@ static struct poptOption long_options[] = { {"delete-delay", 0, POPT_ARG_VAL, &delete_during, 2, 0, 0 }, {"delete-after", 0, POPT_ARG_NONE, &delete_after, 0, 0, 0 }, {"delete-excluded", 0, POPT_ARG_NONE, &delete_excluded, 0, 0, 0 }, - {"delete-missing-args",0,POPT_ARG_NONE, &delete_missing_args, 0, 0, 0 }, {"remove-sent-files",0, POPT_ARG_VAL, &remove_source_files, 2, 0, 0 }, /* deprecated */ {"remove-source-files",0,POPT_ARG_VAL, &remove_source_files, 1, 0, 0 }, {"force", 0, POPT_ARG_VAL, &force_delete, 1, 0, 0 }, @@ -1925,7 +1926,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) return 0; } - if (refused_delete && (delete_mode || delete_missing_args)) { + if (refused_delete && (delete_mode || missing_args == 2)) { create_refuse_error(refused_delete); return 0; } @@ -2482,8 +2483,12 @@ void server_options(char **args, int *argc_p) } } - if (delete_missing_args) + /* --delete-missing-args needs the cooperation of both sides, but + * the sender can handle --ignore-missing-args by itself. */ + if (missing_args == 2) args[ac++] = "--delete-missing-args"; + else if (missing_args == 1 && !am_sender) + args[ac++] = "--ignore-missing-args"; if (modify_window_set) { if (asprintf(&arg, "--modify-window=%d", modify_window) < 0) diff --git a/rsync.yo b/rsync.yo index 2ff224f..317a2ad 100644 --- a/rsync.yo +++ b/rsync.yo @@ -328,6 +328,8 @@ to the detailed description below for a complete description. verb( -r, --recursive recurse into directories -R, --relative use relative path names --no-implied-dirs don't send implied dirs with --relative + --ignore-missing-args ignore args missing in source w/o error + --delete-missing-args delete args missing in source from dest -b, --backup make backups (see --suffix & --backup-dir) --backup-dir=DIR make backups into hierarchy based in DIR --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) @@ -375,7 +377,6 @@ to the detailed description below for a complete description. verb( --delete-delay find deletions during, delete after --delete-after receiver deletes after transfer, not before --delete-excluded also delete excluded files from dest dirs - --delete-missing-args receiver deletes each missing source arg --ignore-errors delete even if there are I/O errors --force force deletion of dirs even if not empty --max-delete=NUM don't delete more than NUM files @@ -709,6 +710,19 @@ When pulling files from an rsync older than 3.0.0, you may need to use this option if the sending side has a symlink in the path you request and you wish the implied directories to be transferred as normal directories. +dit(bf(--ignore-missing-args)) The nonexistence of an explicitly requested +source file (i.e., command-line argument or bf(--files-from) entry) when rsync +first looks for it is normally an error. This option suppresses the error. +It does not affect subsequent vanishings or files discovered through +directory scanning. + +dit(bf(--delete-missing-args)) Like bf(--ignore-missing-args), but also deletes +the destination file (if any) corresponding to each missing source file. +The missing source files are represented by special file-list entries which +show as "*missing" in bf(--list-only) output, so this option is governed by hide +filters and not by protect filters. +This option is independent of any other type of delete processing. + dit(bf(-b, --backup)) With this option, preexisting destination files are renamed as each file is transferred or deleted. You can control where the backup file goes and what (if any) suffix gets appended using the @@ -1264,13 +1278,6 @@ this way on the receiver, and for a way to protect files from bf(--delete-excluded). See bf(--delete) (which is implied) for more details on file-deletion. -dit(bf(--delete-missing-args)) Any source arg that is found to me missing is -treated as a request to delete the file by the receiver rather than the sender -generating a missing-file error. Does not affect vanished files discovered -through recursive scanning of directories, just the args mentioned on the -command-line or in the bf(--files-from) list. Can be used with or without any -other type of delete processing. - dit(bf(--ignore-errors)) Tells bf(--delete) to go ahead and delete files even when there are I/O errors. -- 1.6.2.18.g89f05 From wayned at samba.org Mon Mar 9 08:01:54 2009 From: wayned at samba.org (Wayne Davison) Date: Mon, 9 Mar 2009 08:01:54 -0700 Subject: [Rsync-patches] [PATCH] Give a meaningful error message when we fail to write to a batch file. In-Reply-To: <1233382457.24062.20.camel@mattlaptop2.local> References: <1232836803.10089.2.camel@mattlaptop2.local> <1233382457.24062.20.camel@mattlaptop2.local> Message-ID: <20090309150154.GA12851@blorf.net> On Sat, Jan 31, 2009 at 01:14:17AM -0500, Matt McCutchen wrote: > - if (fd == write_batch_monitor_out) { > - if ((size_t)write(batch_fd, buf, len) != len) > - exit_cleanup(RERR_FILEIO); > - } > + if (fd == write_batch_monitor_out) > + writefd_unbuffered(batch_fd, buf, len); This part concerned me, making me wonder if it would cause problems or not; e.g. if it might try to read more data when writing the batch file. However, I forgot to look into the issue. Have you checked if msg_fd_in is always < 0 when using that to write the batch file? It should always be -1 in the receiver (since it's primary use is in the generator), but the sender uses it during the file-list transfer phase too. ..wayne.. From matt at mattmccutchen.net Mon Mar 9 10:10:57 2009 From: matt at mattmccutchen.net (Matt McCutchen) Date: Mon, 09 Mar 2009 13:10:57 -0400 Subject: [Rsync-patches] [PATCH] Give a meaningful error message when we fail to write to a batch file. In-Reply-To: <20090309150154.GA12851@blorf.net> References: <1232836803.10089.2.camel@mattlaptop2.local> <1233382457.24062.20.camel@mattlaptop2.local> <20090309150154.GA12851@blorf.net> Message-ID: <1236618657.2833.51.camel@mattlaptop2.local> On Mon, 2009-03-09 at 08:01 -0700, Wayne Davison wrote: > On Sat, Jan 31, 2009 at 01:14:17AM -0500, Matt McCutchen wrote: > > - if (fd == write_batch_monitor_out) { > > - if ((size_t)write(batch_fd, buf, len) != len) > > - exit_cleanup(RERR_FILEIO); > > - } > > + if (fd == write_batch_monitor_out) > > + writefd_unbuffered(batch_fd, buf, len); > > This part concerned me, making me wonder if it would cause problems or > not; e.g. if it might try to read more data when writing the batch file. > However, I forgot to look into the issue. Have you checked if msg_fd_in > is always < 0 when using that to write the batch file? It should always > be -1 in the receiver (since it's primary use is in the generator), but > the sender uses it during the file-list transfer phase too. Gosh, I don't know. (You're the one who's supposed to understand the I/O code, not me.) But I don't see how this call to writefd_unbuffered would be any more harmful than the immediately following one that actually sends the data to the other process. -- Matt