1 This patch creates a --delete-during functionality that deletes files on
2 the receiving side incrementally as we traverse the directories. It also
3 defines a --delete-before option, for the traditional way that --delete
6 This patch chooses to make --delete into a synonym for --delete-during, and
7 hide --delete-during, but that can be easily changed if we want to preserve
8 the old functionality (and indeed, the support for popt aliases that is now
9 in CVS will let the user easily choose which method they'd like --delete to
10 invoke, either personally or site-wide, without having to recompile rsync).
12 --- orig/flist.c 2005-01-19 00:24:57
13 +++ flist.c 2005-01-19 00:39:09
14 @@ -45,6 +45,8 @@ extern int cvs_exclude;
17 extern char curr_dir[MAXPATHLEN];
18 +extern char *backup_dir;
19 +extern char *backup_suffix;
20 extern int filesfrom_fd;
22 extern int one_file_system;
23 @@ -57,11 +59,14 @@ extern int preserve_uid;
24 extern int preserve_gid;
25 extern int relative_paths;
26 extern int implied_dirs;
27 +extern int make_backups;
28 +extern int backup_suffix_len;
29 extern int copy_links;
30 extern int copy_unsafe_links;
31 extern int protocol_version;
32 extern int sanitize_paths;
33 extern int delete_excluded;
34 +extern int max_delete;
35 extern int orig_umask;
38 @@ -1044,7 +1049,8 @@ static void send_directory(int f, struct
39 || (dname[1] == '.' && dname[2] == '\0')))
41 if (strlcpy(p, dname, MAXPATHLEN - offset) < MAXPATHLEN - offset) {
42 - send_file_name(f, flist, fname, recurse, 0);
43 + int do_subdirs = recurse >= 1 ? recurse-- : recurse;
44 + send_file_name(f, flist, fname, do_subdirs, 0);
46 io_error |= IOERR_GENERAL;
48 @@ -1102,6 +1108,7 @@ struct file_list *send_file_list(int f,
50 char fname2[MAXPATHLEN];
55 if (read_filesfrom_line(filesfrom_fd, fname) == 0)
56 @@ -1143,7 +1150,7 @@ struct file_list *send_file_list(int f,
60 - if (!relative_paths) {
61 + if (!relative_paths && recurse <= 0) {
62 p = strrchr(fname, '/');
65 @@ -1212,7 +1219,8 @@ struct file_list *send_file_list(int f,
67 set_filesystem(fname);
69 - send_file_name(f, flist, fname, recurse, XMIT_TOP_DIR);
70 + do_subdirs = recurse >= 1 ? recurse-- : recurse;
71 + send_file_name(f, flist, fname, do_subdirs, XMIT_TOP_DIR);
75 @@ -1634,3 +1642,109 @@ char *f_name(struct file_struct *f)
77 return f_name_to(f, names[n]);
80 +static void delete_one(char *fn, int is_dir)
83 + if (robust_unlink(fn) != 0) {
84 + rsyserr(FERROR, errno, "delete_one: unlink %s failed",
87 + rprintf(FINFO, "deleting %s\n", safe_fname(fn));
89 + if (do_rmdir(fn) != 0) {
90 + if (errno != ENOTEMPTY && errno != EEXIST) {
91 + rsyserr(FERROR, errno,
92 + "delete_one: rmdir %s failed",
95 + } else if (verbose) {
96 + rprintf(FINFO, "deleting directory %s\n",
102 +static int is_backup_file(char *fn)
104 + int k = strlen(fn) - backup_suffix_len;
105 + return k > 0 && strcmp(fn+k, backup_suffix) == 0;
108 +void delete_in_dir(struct file_list *flist, char *fname, int do_subdirs)
110 + static int deletion_count = 0;
111 + struct file_list *del_flist;
112 + int save_recurse = recurse;
113 + int save_keep_dirs = keep_dirs;
114 + int save_implied_dirs = implied_dirs;
118 + if (max_delete && deletion_count >= max_delete)
121 + if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
122 + rprintf(FINFO, "IO error encountered - skipping file deletion\n");
123 + max_delete = -1; /* avoid duplicating the above warning */
127 + recurse = do_subdirs ? -1 : 1;
132 + del_flist = send_file_list(-1, 1, argv);
134 + implied_dirs = save_implied_dirs;
135 + keep_dirs = save_keep_dirs;
136 + recurse = save_recurse;
142 + rprintf(FINFO, "deleting in %s\n", safe_fname(fname));
144 + for (i = del_flist->count-1; i >= 0; i--) {
145 + if (max_delete && deletion_count >= max_delete)
147 + if (!del_flist->files[i]->basename)
149 + if (flist_find(flist, del_flist->files[i]) < 0) {
150 + char *f = f_name(del_flist->files[i]);
151 + int mode = del_flist->files[i]->mode;
152 + if (make_backups && (backup_dir || !is_backup_file(f))
153 + && !S_ISDIR(mode)) {
156 + rprintf(FINFO, "deleting %s\n",
160 + delete_one(f, S_ISDIR(mode) != 0);
164 + flist_free(del_flist);
167 +/* This deletes any files on the receiving side that are not present on the
168 + * sending side. This is used by --delete-before and --delete-after. */
169 +void delete_files(struct file_list *flist)
171 + char fbuf[MAXPATHLEN];
175 + add_cvs_excludes();
177 + for (j = 0; j < flist->count; j++) {
178 + if (!(flist->files[j]->flags & FLAG_TOP_DIR)
179 + || !S_ISDIR(flist->files[j]->mode))
182 + delete_in_dir(flist, f_name_to(flist->files[j], fbuf), recurse);
185 --- orig/generator.c 2005-01-17 23:11:45
186 +++ generator.c 2005-01-18 23:14:23
187 @@ -34,6 +34,8 @@ extern int preserve_hard_links;
188 extern int preserve_perms;
189 extern int preserve_uid;
190 extern int preserve_gid;
191 +extern int delete_during;
192 +extern int cvs_exclude;
193 extern int update_only;
194 extern int opt_ignore_existing;
196 @@ -242,7 +244,6 @@ static void generate_and_send_sums(int f
202 * Acts on file number @p i from @p flist, whose name is @p fname.
204 @@ -251,7 +252,8 @@ static void generate_and_send_sums(int f
205 * @note This comment was added later by mbp who was trying to work it
206 * out. It might be wrong.
208 -static void recv_generator(char *fname, struct file_struct *file, int i,
209 +static void recv_generator(char *fname, struct file_list *flist,
210 + struct file_struct *file, int i,
211 int f_out, int f_out_name)
213 int fd = -1, f_copy = -1;
214 @@ -312,8 +314,6 @@ static void recv_generator(char *fname,
215 * we need to delete it. If it doesn't exist, then
216 * recursively create it. */
219 - return; /* TODO: causes inaccuracies -- fix */
220 if (statret == 0 && !S_ISDIR(st.st_mode)) {
221 if (robust_unlink(fname) != 0) {
222 rsyserr(FERROR, errno,
223 @@ -334,6 +334,8 @@ static void recv_generator(char *fname,
225 /* f_out is set to -1 when doing final directory-permission
226 * and modification-time repair. */
227 + if (delete_during && f_out != -1)
228 + delete_in_dir(flist, fname, 0);
229 if (set_perms(fname, file, statret ? NULL : &st, 0)
230 && verbose && f_out != -1)
231 rprintf(FINFO, "%s/\n", safe_fname(fname));
232 @@ -642,6 +644,9 @@ void generate_files(int f_out, struct fi
233 : "delta transmission enabled\n");
236 + if (delete_during && cvs_exclude)
237 + add_cvs_excludes();
239 /* we expect to just sit around now, so don't exit on a
240 timeout. If we really get a timeout then the other process should
242 @@ -665,7 +670,7 @@ void generate_files(int f_out, struct fi
245 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
246 - file, i, f_out, f_out_name);
247 + flist, file, i, f_out, f_out_name);
251 @@ -682,7 +687,7 @@ void generate_files(int f_out, struct fi
252 while ((i = get_redo_num()) != -1) {
253 struct file_struct *file = flist->files[i];
254 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
255 - file, i, f_out, f_out_name);
256 + flist, file, i, f_out, f_out_name);
260 @@ -704,7 +709,7 @@ void generate_files(int f_out, struct fi
261 if (!file->basename || !S_ISDIR(file->mode))
263 recv_generator(local_name ? local_name : f_name(file),
265 + flist, file, i, -1, -1);
269 --- orig/main.c 2005-01-17 23:11:45
270 +++ main.c 2005-01-18 21:56:05
271 @@ -33,6 +33,7 @@ extern int verbose;
272 extern int blocking_io;
273 extern int cvs_exclude;
274 extern int delete_mode;
275 +extern int delete_before;
276 extern int delete_excluded;
277 extern int delete_after;
278 extern int daemon_over_rsh;
279 @@ -473,9 +474,9 @@ static int do_recv(int f_in,int f_out,st
280 if (preserve_hard_links)
281 init_hard_links(flist);
283 - if (!delete_after) {
284 + if (delete_before) {
285 /* I moved this here from recv_files() to prevent a race condition */
286 - if (recurse && delete_mode && !local_name && flist->count > 0)
287 + if (recurse && !local_name && flist->count > 0)
291 --- orig/options.c 2005-01-19 00:34:19
292 +++ options.c 2005-01-19 01:07:34
293 @@ -54,6 +54,8 @@ int dry_run = 0;
294 int local_server = 0;
295 int ignore_times = 0;
297 +int delete_during = 0;
298 +int delete_before = 0;
299 int delete_excluded = 0;
300 int one_file_system = 0;
301 int protocol_version = PROTOCOL_VERSION;
302 @@ -272,14 +274,15 @@ void usage(enum logcode F)
303 rprintf(F," --existing only update files that already exist\n");
304 rprintf(F," --ignore-existing ignore files that already exist on receiving side\n");
305 rprintf(F," --delete delete files that don't exist on the sending side\n");
306 + rprintf(F," --delete-before receiver deletes before transferring, not during\n");
307 + rprintf(F," --delete-after receiver deletes after transferring, not during\n");
308 rprintf(F," --delete-excluded also delete excluded files on the receiving side\n");
309 - rprintf(F," --delete-after receiver deletes after transferring, not before\n");
310 rprintf(F," --ignore-errors delete even if there are I/O errors\n");
311 + rprintf(F," --force force deletion of directories even if not empty\n");
312 rprintf(F," --max-delete=NUM don't delete more than NUM files\n");
313 rprintf(F," --max-size=SIZE don't transfer any file larger than SIZE\n");
314 rprintf(F," --partial keep partially transferred files\n");
315 rprintf(F," --partial-dir=DIR put a partially transferred file into DIR\n");
316 - rprintf(F," --force force deletion of directories even if not empty\n");
317 rprintf(F," --numeric-ids don't map uid/gid values by user/group name\n");
318 rprintf(F," --timeout=TIME set I/O timeout in seconds\n");
319 rprintf(F," -I, --ignore-times turn off mod time & file size quick check\n");
320 @@ -320,8 +323,8 @@ void usage(enum logcode F)
321 rprintf(F,"See http://rsync.samba.org/ for updates, bug reports, and answers\n");
324 -enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
325 - OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED,
326 +enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE,
327 + OPT_EXCLUDE_FROM, OPT_DELETE_MODE,
328 OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST,
329 OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
330 OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT, OPT_MAX_SIZE,
331 @@ -337,11 +340,13 @@ static struct poptOption long_options[]
332 {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 },
333 {"modify-window", 0, POPT_ARG_INT, &modify_window, OPT_MODIFY_WINDOW, 0, 0 },
334 {"one-file-system", 'x', POPT_ARG_NONE, &one_file_system, 0, 0, 0 },
335 - {"delete", 0, POPT_ARG_NONE, &delete_mode, 0, 0, 0 },
336 + {"delete", 0, POPT_ARG_NONE, 0, OPT_DELETE_MODE, 0, 0 },
337 + {"delete-during", 0, POPT_ARG_NONE, &delete_during, OPT_DELETE_MODE, 0, 0 },
338 + {"delete-before", 0, POPT_ARG_NONE, &delete_before, OPT_DELETE_MODE, 0, 0 },
339 {"existing", 0, POPT_ARG_NONE, &only_existing, 0, 0, 0 },
340 {"ignore-existing", 0, POPT_ARG_NONE, &opt_ignore_existing, 0, 0, 0 },
341 - {"delete-after", 0, POPT_ARG_NONE, 0, OPT_DELETE_AFTER, 0, 0 },
342 - {"delete-excluded", 0, POPT_ARG_NONE, 0, OPT_DELETE_EXCLUDED, 0, 0 },
343 + {"delete-after", 0, POPT_ARG_NONE, &delete_after, OPT_DELETE_MODE, 0, 0 },
344 + {"delete-excluded", 0, POPT_ARG_NONE, &delete_excluded, OPT_DELETE_MODE, 0, 0 },
345 {"force", 0, POPT_ARG_NONE, &force_delete, 0, 0, 0 },
346 {"numeric-ids", 0, POPT_ARG_NONE, &numeric_ids, 0, 0, 0 },
347 {"exclude", 0, POPT_ARG_STRING, 0, OPT_EXCLUDE, 0, 0 },
348 @@ -613,13 +618,7 @@ int parse_arguments(int *argc, const cha
349 modify_window_set = 1;
352 - case OPT_DELETE_AFTER:
357 - case OPT_DELETE_EXCLUDED:
358 - delete_excluded = 1;
359 + case OPT_DELETE_MODE:
363 @@ -851,13 +850,18 @@ int parse_arguments(int *argc, const cha
365 preserve_devices = 1;
369 + recurse = -1; /* unlimited recursion */
373 if (relative_paths < 0)
374 relative_paths = files_from? 1 : 0;
376 + if (delete_mode && !(delete_during || delete_before || delete_after))
377 + delete_during = 1; /* Use "delete_before" for old default. */
379 *argv = poptGetArgs(pc);
380 *argc = count_args(*argv);
382 @@ -1156,7 +1160,9 @@ void server_options(char **args,int *arg
385 args[ac++] = "--delete-excluded";
386 - else if (delete_mode)
387 + else if (delete_before)
388 + args[ac++] = "--delete-before";
389 + else if (delete_during || delete_after)
390 args[ac++] = "--delete";
393 --- orig/receiver.c 2005-01-17 23:11:45
394 +++ receiver.c 2005-01-18 22:47:38
398 extern int delete_after;
399 -extern int max_delete;
400 extern int csum_length;
401 extern struct stats stats;
403 @@ -35,7 +34,6 @@ extern int relative_paths;
404 extern int keep_dirlinks;
405 extern int preserve_hard_links;
406 extern int preserve_perms;
407 -extern int cvs_exclude;
410 extern char *partial_dir;
411 @@ -43,9 +41,6 @@ extern char *basis_dir[];
412 extern int basis_dir_cnt;
413 extern int make_backups;
414 extern int do_progress;
415 -extern char *backup_dir;
416 -extern char *backup_suffix;
417 -extern int backup_suffix_len;
418 extern int cleanup_got_literal;
419 extern int module_id;
420 extern int ignore_errors;
421 @@ -57,91 +52,6 @@ extern int inplace;
422 extern struct exclude_list_struct server_exclude_list;
425 -static void delete_one(char *fn, int is_dir)
428 - if (robust_unlink(fn) != 0) {
429 - rsyserr(FERROR, errno, "delete_one: unlink %s failed",
431 - } else if (verbose)
432 - rprintf(FINFO, "deleting %s\n", safe_fname(fn));
434 - if (do_rmdir(fn) != 0) {
435 - if (errno != ENOTEMPTY && errno != EEXIST) {
436 - rsyserr(FERROR, errno,
437 - "delete_one: rmdir %s failed",
440 - } else if (verbose) {
441 - rprintf(FINFO, "deleting directory %s\n",
448 -static int is_backup_file(char *fn)
450 - int k = strlen(fn) - backup_suffix_len;
451 - return k > 0 && strcmp(fn+k, backup_suffix) == 0;
455 -/* This deletes any files on the receiving side that are not present
456 - * on the sending side. */
457 -void delete_files(struct file_list *flist)
459 - struct file_list *local_file_list;
461 - char *argv[1], fbuf[MAXPATHLEN];
462 - static int deletion_count;
465 - add_cvs_excludes();
467 - if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
468 - rprintf(FINFO,"IO error encountered - skipping file deletion\n");
472 - for (j = 0; j < flist->count; j++) {
473 - if (!(flist->files[j]->flags & FLAG_TOP_DIR)
474 - || !S_ISDIR(flist->files[j]->mode))
477 - argv[0] = f_name_to(flist->files[j], fbuf);
479 - if (!(local_file_list = send_file_list(-1, 1, argv)))
483 - rprintf(FINFO, "deleting in %s\n", safe_fname(fbuf));
485 - for (i = local_file_list->count-1; i >= 0; i--) {
486 - if (max_delete && deletion_count >= max_delete)
488 - if (!local_file_list->files[i]->basename)
490 - if (flist_find(flist,local_file_list->files[i]) < 0) {
491 - char *f = f_name(local_file_list->files[i]);
492 - int mode = local_file_list->files[i]->mode;
493 - if (make_backups && (backup_dir || !is_backup_file(f))
494 - && !S_ISDIR(mode)) {
497 - rprintf(FINFO, "deleting %s\n",
501 - delete_one(f, S_ISDIR(mode) != 0);
505 - flist_free(local_file_list);
511 * get_tmpname() - create a tmp filename for a given filename
513 --- orig/rsync.yo 2005-01-17 23:11:46
514 +++ rsync.yo 2005-01-19 01:05:05
515 @@ -341,14 +341,15 @@ verb(
516 --existing only update files that already exist
517 --ignore-existing ignore files that already exist on receiver
518 --delete delete files that don't exist on sender
519 + --delete-before receiver deletes before xfer, not during
520 + --delete-after receiver deletes after transfer, not during
521 --delete-excluded also delete excluded files on receiver
522 - --delete-after receiver deletes after transfer, not before
523 --ignore-errors delete even if there are I/O errors
524 + --force force deletion of dirs even if not empty
525 --max-delete=NUM don't delete more than NUM files
526 --max-size=SIZE don't transfer any file larger than SIZE
527 --partial keep partially transferred files
528 --partial-dir=DIR put a partially transferred file into DIR
529 - --force force deletion of dirs even if not empty
530 --numeric-ids don't map uid/gid values by user/group name
531 --timeout=TIME set I/O timeout in seconds
532 -I, --ignore-times turn off mod time & file size quick check
533 @@ -669,7 +670,7 @@ by the shell and rsync thus gets a reque
534 the files' parent directory. Files that are excluded from transfer are
535 excluded from being deleted unless you use --delete-excluded.
537 -This option has no effect if directory recursion is not selected.
538 +This option has no effect unless directory recursion is selected.
540 This option can be dangerous if used incorrectly! It is a very good idea
541 to run first using the --dry-run option (-n) to see what files would be
542 @@ -681,20 +682,29 @@ prevent temporary filesystem failures (s
543 sending side causing a massive deletion of files on the
544 destination. You can override this with the --ignore-errors option.
546 +By default rsync does file deletions on the receiving side during the
547 +transfer of files to try make it as efficient as possible. For other
548 +options, see --delete-before and --delte-after.
550 +dit(bf(--delete-before)) Request that the file-deletions on the receving
551 +side be done prior to starting the transfer, not incrementally as the
552 +transfer happens. Implies --delete.
554 +One reason to use --delete-before is if the filesystem is tight for space
555 +and removing extraneous files would help to make the transfer possible.
556 +However, it does introduce a delay before the start of the transfer (while
557 +the receiving side is being scanned for deletions) and this delay might
558 +cause the transfer to timeout.
560 +dit(bf(--delete-after)) Request that the file-deletions on the receving
561 +side be done after the transfer has completed, not incrementally as the
562 +transfer happens. Implies --delete.
564 dit(bf(--delete-excluded)) In addition to deleting the files on the
565 receiving side that are not on the sending side, this tells rsync to also
566 delete any files on the receiving side that are excluded (see --exclude).
569 -dit(bf(--delete-after)) By default rsync does file deletions on the
570 -receiving side before transferring files to try to ensure that there is
571 -sufficient space on the receiving filesystem. If you want to delete
572 -after transferring, use the --delete-after switch. Implies --delete.
574 -One reason to use --delete-after is to avoid a delay before the start of
575 -the transfer (while the receiving side is scanned for deletions) as this
576 -delay might cause the transfer to timeout.
578 dit(bf(--ignore-errors)) Tells --delete to go ahead and delete files
579 even when there are I/O errors.