1 This patch creates a --delete-during functionality that deletes files on
2 the receiving side incrementally as we traverse the directories. It also
3 defines a --delete-before option, for the traditional way that --delete
6 This patch chooses to make --delete into a synonym for --delete-during, and
7 hide --delete-during, but that can be easily changed if we want to preserve
8 the old functionality (and indeed, the support for popt aliases that is now
9 in CVS will let the user easily choose which method they'd like --delete to
10 invoke, either personally or site-wide, without having to recompile rsync).
12 Be sure to run "make proto" before "make".
14 --- orig/flist.c 2005-01-21 00:35:26
15 +++ flist.c 2005-01-21 10:16:02
16 @@ -35,6 +35,7 @@ extern int am_root;
20 +extern int delete_during;
21 extern int always_checksum;
23 extern int ignore_errors;
24 @@ -45,6 +46,8 @@ extern int cvs_exclude;
27 extern char curr_dir[MAXPATHLEN];
28 +extern char *backup_dir;
29 +extern char *backup_suffix;
30 extern int filesfrom_fd;
32 extern int one_file_system;
33 @@ -57,11 +60,14 @@ extern int preserve_uid;
34 extern int preserve_gid;
35 extern int relative_paths;
36 extern int implied_dirs;
37 +extern int make_backups;
38 +extern int backup_suffix_len;
39 extern int copy_links;
40 extern int copy_unsafe_links;
41 extern int protocol_version;
42 extern int sanitize_paths;
43 extern int delete_excluded;
44 +extern int max_delete;
45 extern int orig_umask;
48 @@ -534,6 +540,8 @@ void receive_file_entry(struct file_stru
50 static char lastname[MAXPATHLEN], *lastdir;
51 static int lastdir_depth, lastdir_len = -1;
52 + static unsigned int del_heir_name_len = -1;
53 + static int in_del_hier = 0;
54 char thisname[MAXPATHLEN];
55 unsigned int l1 = 0, l2 = 0;
56 int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
57 @@ -547,7 +555,8 @@ void receive_file_entry(struct file_stru
62 + del_heir_name_len = lastdir_len = -1;
67 @@ -645,13 +654,27 @@ void receive_file_entry(struct file_stru
68 memset(bp, 0, file_struct_len);
69 bp += file_struct_len;
71 - file->flags = flags & XMIT_DEL_START ? FLAG_DEL_START : 0;
73 file->modtime = modtime;
74 file->length = file_length;
79 + if (S_ISDIR(mode)) {
80 + if (flags & XMIT_DEL_START) {
82 + del_heir_name_len = l1 + l2;
83 + file->flags |= FLAG_DEL_START;
84 + } else if (delete_during && in_del_hier) {
85 + if (!relative_paths || (l1 >= del_heir_name_len
86 + && thisname[del_heir_name_len] == '/'))
87 + file->flags |= FLAG_DEL_START;
94 file->dirname = lastdir = bp;
95 lastdir_len = dirname_len - 1;
96 @@ -1044,7 +1067,8 @@ static void send_directory(int f, struct
97 || (dname[1] == '.' && dname[2] == '\0')))
99 if (strlcpy(p, dname, MAXPATHLEN - offset) < MAXPATHLEN - offset) {
100 - send_file_name(f, flist, fname, recurse, 0);
101 + int do_subdirs = recurse >= 1 ? recurse-- : recurse;
102 + send_file_name(f, flist, fname, do_subdirs, 0);
104 io_error |= IOERR_GENERAL;
106 @@ -1102,6 +1126,7 @@ struct file_list *send_file_list(int f,
108 char fname2[MAXPATHLEN];
109 char *fname = fname2;
113 if (read_filesfrom_line(filesfrom_fd, fname) == 0)
114 @@ -1143,7 +1168,7 @@ struct file_list *send_file_list(int f,
118 - if (!relative_paths) {
119 + if (!relative_paths && recurse <= 0) {
120 p = strrchr(fname, '/');
123 @@ -1212,7 +1237,8 @@ struct file_list *send_file_list(int f,
125 set_filesystem(fname);
127 - send_file_name(f, flist, fname, recurse, XMIT_DEL_START);
128 + do_subdirs = recurse >= 1 ? recurse-- : recurse;
129 + send_file_name(f, flist, fname, do_subdirs, XMIT_DEL_START);
133 @@ -1634,3 +1660,72 @@ char *f_name(struct file_struct *f)
135 return f_name_to(f, names[n]);
138 +static int is_backup_file(char *fn)
140 + int k = strlen(fn) - backup_suffix_len;
141 + return k > 0 && strcmp(fn+k, backup_suffix) == 0;
144 +void delete_in_dir(struct file_list *flist, char *fname, int do_subdirs)
146 + static int deletion_count = 0;
147 + struct file_list *del_flist;
148 + int save_recurse = recurse;
149 + int save_keep_dirs = keep_dirs;
150 + int save_implied_dirs = implied_dirs;
154 + if (max_delete && deletion_count >= max_delete)
157 + if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
158 + rprintf(FINFO, "IO error encountered - skipping file deletion\n");
159 + max_delete = -1; /* avoid duplicating the above warning */
163 + recurse = do_subdirs ? -1 : 1;
168 + del_flist = send_file_list(-1, 1, argv);
170 + implied_dirs = save_implied_dirs;
171 + keep_dirs = save_keep_dirs;
172 + recurse = save_recurse;
178 + rprintf(FINFO, "deleting in %s\n", safe_fname(fname));
180 + for (i = del_flist->count-1; i >= 0; i--) {
181 + if (max_delete && deletion_count >= max_delete)
183 + if (!del_flist->files[i]->basename)
185 + mode = del_flist->files[i]->mode;
186 + if ((j = flist_find(flist, del_flist->files[i])) < 0
187 + || (delete_during && S_ISDIR(mode)
188 + && !S_ISDIR(flist->files[j]->mode))) {
189 + char *f = f_name(del_flist->files[i]);
190 + if (make_backups && (backup_dir || !is_backup_file(f))
191 + && !S_ISDIR(mode)) {
194 + rprintf(FINFO, "deleting %s\n",
198 + delete_file(f, S_ISDIR(mode)
199 + ? DEL_DIR | DEL_RECURSE : 0);
204 + flist_free(del_flist);
206 --- orig/generator.c 2005-01-20 23:05:34
207 +++ generator.c 2005-01-21 10:21:39
208 @@ -34,6 +34,7 @@ extern int preserve_hard_links;
209 extern int preserve_perms;
210 extern int preserve_uid;
211 extern int preserve_gid;
212 +extern int delete_during;
213 extern int update_only;
214 extern int opt_ignore_existing;
216 @@ -233,7 +234,8 @@ static void generate_and_send_sums(int f
217 * @note This comment was added later by mbp who was trying to work it
218 * out. It might be wrong.
220 -static void recv_generator(char *fname, struct file_struct *file, int i,
221 +static void recv_generator(char *fname, struct file_list *flist,
222 + struct file_struct *file, int i,
223 int f_out, int f_out_name)
225 int fd = -1, f_copy = -1;
226 @@ -308,6 +310,11 @@ static void recv_generator(char *fname,
228 /* f_out is set to -1 when doing final directory-permission
229 * and modification-time repair. */
230 + if (delete_during && f_out != -1
231 + && (file->flags & FLAG_DEL_START)) {
232 + delete_in_dir(flist, fname, 0);
235 if (set_perms(fname, file, statret ? NULL : &st, 0)
236 && verbose && f_out != -1)
237 rprintf(FINFO, "%s/\n", safe_fname(fname));
238 @@ -641,7 +648,7 @@ void generate_files(int f_out, struct fi
241 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
242 - file, i, f_out, f_out_name);
243 + flist, file, i, f_out, f_out_name);
247 @@ -658,7 +665,7 @@ void generate_files(int f_out, struct fi
248 while ((i = get_redo_num()) != -1) {
249 struct file_struct *file = flist->files[i];
250 recv_generator(local_name ? local_name : f_name_to(file, fbuf),
251 - file, i, f_out, f_out_name);
252 + flist, file, i, f_out, f_out_name);
256 @@ -680,7 +687,7 @@ void generate_files(int f_out, struct fi
257 if (!file->basename || !S_ISDIR(file->mode))
259 recv_generator(local_name ? local_name : f_name(file),
261 + flist, file, i, -1, -1);
265 --- orig/main.c 2005-01-17 23:11:45
266 +++ main.c 2005-01-21 10:55:05
267 @@ -33,6 +33,7 @@ extern int verbose;
268 extern int blocking_io;
269 extern int cvs_exclude;
270 extern int delete_mode;
271 +extern int delete_before;
272 extern int delete_excluded;
273 extern int delete_after;
274 extern int daemon_over_rsh;
275 @@ -473,9 +474,9 @@ static int do_recv(int f_in,int f_out,st
276 if (preserve_hard_links)
277 init_hard_links(flist);
279 - if (!delete_after) {
280 + if (delete_before) {
281 /* I moved this here from recv_files() to prevent a race condition */
282 - if (recurse && delete_mode && !local_name && flist->count > 0)
283 + if (recurse && !local_name && flist->count > 0)
287 @@ -735,6 +736,8 @@ int client_run(int f_in, int f_out, pid_
290 send_exclude_list(f_out);
292 + add_cvs_excludes();
294 if (filesfrom_fd >= 0) {
295 io_set_filesfrom_fds(filesfrom_fd, f_out);
296 --- orig/options.c 2005-01-20 23:05:34
297 +++ options.c 2005-01-20 23:06:33
298 @@ -54,6 +54,8 @@ int dry_run = 0;
299 int local_server = 0;
300 int ignore_times = 0;
302 +int delete_during = 0;
303 +int delete_before = 0;
304 int delete_excluded = 0;
305 int one_file_system = 0;
306 int protocol_version = PROTOCOL_VERSION;
307 @@ -272,14 +274,15 @@ void usage(enum logcode F)
308 rprintf(F," --existing only update files that already exist\n");
309 rprintf(F," --ignore-existing ignore files that already exist on receiving side\n");
310 rprintf(F," --delete delete files that don't exist on the sending side\n");
311 + rprintf(F," --delete-before receiver deletes before transfer, not during\n");
312 + rprintf(F," --delete-after receiver deletes after transfer, not during\n");
313 rprintf(F," --delete-excluded also delete excluded files on the receiving side\n");
314 - rprintf(F," --delete-after receiver deletes after transferring, not before\n");
315 rprintf(F," --ignore-errors delete even if there are I/O errors\n");
316 + rprintf(F," --force force deletion of directories even if not empty\n");
317 rprintf(F," --max-delete=NUM don't delete more than NUM files\n");
318 rprintf(F," --max-size=SIZE don't transfer any file larger than SIZE\n");
319 rprintf(F," --partial keep partially transferred files\n");
320 rprintf(F," --partial-dir=DIR put a partially transferred file into DIR\n");
321 - rprintf(F," --force force deletion of directories even if not empty\n");
322 rprintf(F," --numeric-ids don't map uid/gid values by user/group name\n");
323 rprintf(F," --timeout=TIME set I/O timeout in seconds\n");
324 rprintf(F," -I, --ignore-times turn off mod time & file size quick check\n");
325 @@ -321,7 +324,6 @@ void usage(enum logcode F)
328 enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
329 - OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED,
330 OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST,
331 OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
332 OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT, OPT_MAX_SIZE,
333 @@ -337,11 +339,13 @@ static struct poptOption long_options[]
334 {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 },
335 {"modify-window", 0, POPT_ARG_INT, &modify_window, OPT_MODIFY_WINDOW, 0, 0 },
336 {"one-file-system", 'x', POPT_ARG_NONE, &one_file_system, 0, 0, 0 },
337 - {"delete", 0, POPT_ARG_NONE, &delete_mode, 0, 0, 0 },
338 + {"delete", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 },
339 + {"delete-during", 0, POPT_ARG_NONE, &delete_during, 0, 0, 0 },
340 + {"delete-before", 0, POPT_ARG_NONE, &delete_before, 0, 0, 0 },
341 {"existing", 0, POPT_ARG_NONE, &only_existing, 0, 0, 0 },
342 {"ignore-existing", 0, POPT_ARG_NONE, &opt_ignore_existing, 0, 0, 0 },
343 - {"delete-after", 0, POPT_ARG_NONE, 0, OPT_DELETE_AFTER, 0, 0 },
344 - {"delete-excluded", 0, POPT_ARG_NONE, 0, OPT_DELETE_EXCLUDED, 0, 0 },
345 + {"delete-after", 0, POPT_ARG_NONE, &delete_after, 0, 0, 0 },
346 + {"delete-excluded", 0, POPT_ARG_NONE, &delete_excluded, 0, 0, 0 },
347 {"force", 0, POPT_ARG_NONE, &force_delete, 0, 0, 0 },
348 {"numeric-ids", 0, POPT_ARG_NONE, &numeric_ids, 0, 0, 0 },
349 {"exclude", 0, POPT_ARG_STRING, 0, OPT_EXCLUDE, 0, 0 },
350 @@ -613,16 +617,6 @@ int parse_arguments(int *argc, const cha
351 modify_window_set = 1;
354 - case OPT_DELETE_AFTER:
359 - case OPT_DELETE_EXCLUDED:
360 - delete_excluded = 1;
365 add_exclude(&exclude_list, poptGetOptArg(pc), 0);
367 @@ -851,13 +845,20 @@ int parse_arguments(int *argc, const cha
369 preserve_devices = 1;
373 + recurse = -1; /* unlimited recursion */
377 if (relative_paths < 0)
378 relative_paths = files_from? 1 : 0;
380 + if (delete_during || delete_before || delete_after)
382 + if (delete_excluded && !delete_mode)
383 + delete_mode = delete_during = 1;
385 *argv = poptGetArgs(pc);
386 *argc = count_args(*argv);
388 @@ -1156,7 +1157,9 @@ void server_options(char **args,int *arg
391 args[ac++] = "--delete-excluded";
392 - else if (delete_mode)
393 + else if (delete_before)
394 + args[ac++] = "--delete-before";
395 + else if (delete_during || delete_after)
396 args[ac++] = "--delete";
399 --- orig/receiver.c 2005-01-21 00:35:26
400 +++ receiver.c 2005-01-21 10:22:37
404 extern int delete_after;
405 -extern int max_delete;
406 extern int csum_length;
407 extern struct stats stats;
409 @@ -35,7 +34,6 @@ extern int relative_paths;
410 extern int keep_dirlinks;
411 extern int preserve_hard_links;
412 extern int preserve_perms;
413 -extern int cvs_exclude;
416 extern char *partial_dir;
417 @@ -43,9 +41,6 @@ extern char *basis_dir[];
418 extern int basis_dir_cnt;
419 extern int make_backups;
420 extern int do_progress;
421 -extern char *backup_dir;
422 -extern char *backup_suffix;
423 -extern int backup_suffix_len;
424 extern int cleanup_got_literal;
425 extern int module_id;
426 extern int ignore_errors;
427 @@ -57,66 +52,19 @@ extern int inplace;
428 extern struct exclude_list_struct server_exclude_list;
431 -static int is_backup_file(char *fn)
433 - int k = strlen(fn) - backup_suffix_len;
434 - return k > 0 && strcmp(fn+k, backup_suffix) == 0;
438 -/* This deletes any files on the receiving side that are not present
439 - * on the sending side. */
440 +/* This deletes any files on the receiving side that are not present on the
441 + * sending side. This is used by --delete-before and --delete-after. */
442 void delete_files(struct file_list *flist)
444 - struct file_list *local_file_list;
446 - char *argv[1], fbuf[MAXPATHLEN];
447 - static int deletion_count;
450 - add_cvs_excludes();
452 - if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
453 - rprintf(FINFO,"IO error encountered - skipping file deletion\n");
456 + char fbuf[MAXPATHLEN];
459 for (j = 0; j < flist->count; j++) {
460 if (!(flist->files[j]->flags & FLAG_DEL_START)
461 || !S_ISDIR(flist->files[j]->mode))
464 - argv[0] = f_name_to(flist->files[j], fbuf);
466 - if (!(local_file_list = send_file_list(-1, 1, argv)))
470 - rprintf(FINFO, "deleting in %s\n", safe_fname(fbuf));
472 - for (i = local_file_list->count-1; i >= 0; i--) {
473 - if (max_delete && deletion_count >= max_delete)
475 - if (!local_file_list->files[i]->basename)
477 - if (flist_find(flist,local_file_list->files[i]) < 0) {
478 - char *f = f_name(local_file_list->files[i]);
479 - int mode = local_file_list->files[i]->mode;
480 - if (make_backups && (backup_dir || !is_backup_file(f))
481 - && !S_ISDIR(mode)) {
484 - rprintf(FINFO, "deleting %s\n",
488 - delete_file(f, S_ISDIR(mode)
489 - ? DEL_DIR | DEL_RECURSE : 0);
494 - flist_free(local_file_list);
495 + delete_in_dir(flist, f_name_to(flist->files[j], fbuf), recurse);
499 --- orig/rsync.yo 2005-01-20 19:47:08
500 +++ rsync.yo 2005-01-19 01:05:05
501 @@ -341,14 +341,15 @@ verb(
502 --existing only update files that already exist
503 --ignore-existing ignore files that already exist on receiver
504 --delete delete files that don't exist on sender
505 + --delete-before receiver deletes before xfer, not during
506 + --delete-after receiver deletes after transfer, not during
507 --delete-excluded also delete excluded files on receiver
508 - --delete-after receiver deletes after transfer, not before
509 --ignore-errors delete even if there are I/O errors
510 + --force force deletion of dirs even if not empty
511 --max-delete=NUM don't delete more than NUM files
512 --max-size=SIZE don't transfer any file larger than SIZE
513 --partial keep partially transferred files
514 --partial-dir=DIR put a partially transferred file into DIR
515 - --force force deletion of dirs even if not empty
516 --numeric-ids don't map uid/gid values by user/group name
517 --timeout=TIME set I/O timeout in seconds
518 -I, --ignore-times turn off mod time & file size quick check
519 @@ -669,7 +670,7 @@ by the shell and rsync thus gets a reque
520 the files' parent directory. Files that are excluded from transfer are
521 excluded from being deleted unless you use --delete-excluded.
523 -This option has no effect if directory recursion is not selected.
524 +This option has no effect unless directory recursion is selected.
526 This option can be dangerous if used incorrectly! It is a very good idea
527 to run first using the --dry-run option (-n) to see what files would be
528 @@ -681,20 +682,29 @@ prevent temporary filesystem failures (s
529 sending side causing a massive deletion of files on the
530 destination. You can override this with the --ignore-errors option.
532 +By default rsync does file deletions on the receiving side during the
533 +transfer of files to try make it as efficient as possible. For other
534 +options, see --delete-before and --delte-after.
536 +dit(bf(--delete-before)) Request that the file-deletions on the receving
537 +side be done prior to starting the transfer, not incrementally as the
538 +transfer happens. Implies --delete.
540 +One reason to use --delete-before is if the filesystem is tight for space
541 +and removing extraneous files would help to make the transfer possible.
542 +However, it does introduce a delay before the start of the transfer (while
543 +the receiving side is being scanned for deletions) and this delay might
544 +cause the transfer to timeout.
546 +dit(bf(--delete-after)) Request that the file-deletions on the receving
547 +side be done after the transfer has completed, not incrementally as the
548 +transfer happens. Implies --delete.
550 dit(bf(--delete-excluded)) In addition to deleting the files on the
551 receiving side that are not on the sending side, this tells rsync to also
552 delete any files on the receiving side that are excluded (see --exclude).
555 -dit(bf(--delete-after)) By default rsync does file deletions on the
556 -receiving side before transferring files to try to ensure that there is
557 -sufficient space on the receiving filesystem. If you want to delete
558 -after transferring, use the --delete-after switch. Implies --delete.
560 -One reason to use --delete-after is to avoid a delay before the start of
561 -the transfer (while the receiving side is scanned for deletions) as this
562 -delay might cause the transfer to timeout.
564 dit(bf(--ignore-errors)) Tells --delete to go ahead and delete files
565 even when there are I/O errors.