From f3d6d4800b35fd65d2395defdb57fed16cea3e85 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 28 Dec 2006 07:54:31 +0000 Subject: [PATCH] Handle the new incremental-recursion mode. --- generator.c | 393 ++++++++++++++++++++++++++++++++-------------------- main.c | 96 +++++++------ receiver.c | 98 +++++++------ rsync.c | 105 +++++++++++--- sender.c | 140 +++++++++++-------- 5 files changed, 524 insertions(+), 308 deletions(-) diff --git a/generator.c b/generator.c index f17faed5..80ce378d 100644 --- a/generator.c +++ b/generator.c @@ -31,6 +31,7 @@ extern int logfile_format_has_i; extern int am_root; extern int am_server; extern int am_daemon; +extern int incremental; extern int do_progress; extern int relative_paths; extern int implied_dirs; @@ -48,7 +49,7 @@ extern int delete_mode; extern int delete_before; extern int delete_during; extern int delete_after; -extern int module_id; +extern int done_cnt; extern int ignore_errors; extern int remove_source_files; extern int delay_updates; @@ -64,10 +65,12 @@ extern int size_only; extern OFF_T max_size; extern OFF_T min_size; extern int io_error; +extern int flist_eof; extern int allowed_lull; extern int sock_f_out; extern int ignore_timeout; extern int protocol_version; +extern int file_total; extern int fuzzy_basis; extern int always_checksum; extern int checksum_len; @@ -90,13 +93,14 @@ extern dev_t filesystem_dev; extern char *backup_dir; extern char *backup_suffix; extern int backup_suffix_len; -extern struct file_list *the_file_list; +extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct filter_list_struct server_filter_list; int ignore_perishable = 0; int non_perishable_cnt = 0; int maybe_ATTRS_REPORT = 0; +static dev_t dev_zero; static int deletion_count = 0; /* used to implement --max-delete */ static int deldelay_size = 0, deldelay_cnt = 0; static char *deldelay_buf = NULL; @@ -158,7 +162,7 @@ static enum delret delete_item(char *fbuf, int mode, char *replace, int flags) if (S_ISDIR(mode)) { what = "rmdir"; ok = do_rmdir(fbuf) == 0; - } else if (make_backups && (backup_dir || !is_backup_file(fbuf))) { + } else if (make_backups > 0 && (backup_dir || !is_backup_file(fbuf))) { what = "make_backup"; ok = make_backup(fbuf); } else { @@ -407,20 +411,15 @@ static void do_delayed_deletions(char *delbuf) * call will append names onto the end, but the old dir value will be restored * on exit). */ static void delete_in_dir(struct file_list *flist, char *fbuf, - struct file_struct *file, STRUCT_STAT *stp) + struct file_struct *file, dev_t *fs_dev) { - static int min_depth = MAXPATHLEN, cur_depth = -1; - static void *filt_array[MAXPATHLEN/2+1]; static int already_warned = 0; struct file_list *dirlist; char delbuf[MAXPATHLEN]; int dlen, i; if (!flist) { - while (cur_depth >= min_depth) - pop_local_filters(filt_array[cur_depth--]); - min_depth = MAXPATHLEN; - cur_depth = -1; + change_local_filter_dir(NULL, 0, 0); return; } @@ -430,9 +429,6 @@ static void delete_in_dir(struct file_list *flist, char *fbuf, if (allowed_lull) maybe_send_keepalive(); - if (F_DEPTH(file) >= MAXPATHLEN/2+1) - return; /* Impossible... */ - if (io_error && !ignore_errors) { if (already_warned) return; @@ -442,18 +438,13 @@ static void delete_in_dir(struct file_list *flist, char *fbuf, return; } - while (cur_depth >= F_DEPTH(file) && cur_depth >= min_depth) - pop_local_filters(filt_array[cur_depth--]); - cur_depth = F_DEPTH(file); - if (min_depth > cur_depth) - min_depth = cur_depth; dlen = strlen(fbuf); - filt_array[cur_depth] = push_local_filters(fbuf, dlen); + change_local_filter_dir(fbuf, dlen, F_DEPTH(file)); if (one_file_system) { if (file->flags & FLAG_TOP_DIR) - filesystem_dev = stp->st_dev; - else if (filesystem_dev != stp->st_dev) + filesystem_dev = *fs_dev; + else if (filesystem_dev != *fs_dev) return; } @@ -510,9 +501,9 @@ static void do_delete_pass(struct file_list *flist) || !S_ISDIR(st.st_mode)) continue; - delete_in_dir(flist, fbuf, file, &st); + delete_in_dir(flist, fbuf, file, &st.st_dev); } - delete_in_dir(NULL, NULL, NULL, NULL); + delete_in_dir(NULL, NULL, NULL, &dev_zero); if (do_progress && !am_server) rprintf(FINFO, " \r"); @@ -532,8 +523,9 @@ int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st) return 1; } -void itemize(struct file_struct *file, int ndx, int statret, STRUCT_STAT *st, - int32 iflags, uchar fnamecmp_type, const char *xname) +void itemize(struct file_struct *file, int ndx, int statret, + STRUCT_STAT *st, int32 iflags, uchar fnamecmp_type, + const char *xname) { if (statret >= 0) { /* A from-dest-dir statret can == 1! */ int keep_time = !preserve_times ? 0 @@ -562,7 +554,7 @@ void itemize(struct file_struct *file, int ndx, int statret, STRUCT_STAT *st, || stdout_format_has_i > 1 || (xname && *xname)) && !read_batch) { if (protocol_version >= 29) { if (ndx >= 0) - write_int(sock_f_out, ndx); + write_int(sock_f_out, ndx + cur_flist->ndx_start); write_shortint(sock_f_out, iflags); if (iflags & ITEM_BASIS_TYPE_FOLLOWS) write_byte(sock_f_out, fnamecmp_type); @@ -584,13 +576,13 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) /* if always checksum is set then we use the checksum instead of the file time to determine whether to sync */ - if (always_checksum && S_ISREG(st->st_mode)) { + if (always_checksum > 0 && S_ISREG(st->st_mode)) { char sum[MD4_SUM_LENGTH]; file_checksum(fn, sum, st->st_size); return memcmp(sum, F_SUM(file), checksum_len) == 0; } - if (size_only) + if (size_only > 0) return 1; if (ignore_times) @@ -749,7 +741,8 @@ static int find_fuzzy(struct file_struct *file, struct file_list *dirlist) int len, suf_len; uint32 dist; - if (!S_ISREG(fp->mode) || !F_LENGTH(fp) || fp->flags & FLAG_SENT) + if (!S_ISREG(fp->mode) || !F_LENGTH(fp) + || fp->flags & FLAG_FILE_SENT) continue; name = fp->basename; @@ -788,17 +781,15 @@ static int find_fuzzy(struct file_struct *file, struct file_list *dirlist) void check_for_finished_hlinks(int itemizing, enum logcode code) { struct file_struct *file; + struct file_list *flist; char fbuf[MAXPATHLEN]; int ndx; while ((ndx = get_hlink_num()) != -1) { - if (ndx < 0 || ndx >= the_file_list->count) - continue; - - file = the_file_list->files[ndx]; - if (!F_IS_HLINKED(file)) - continue; - + flist = flist_for_ndx(ndx); + assert(flist != NULL); + file = flist->files[ndx]; + assert(file->flags & FLAG_HLINKED); finish_hard_link(file, f_name(file, fbuf), NULL, itemizing, code, -1); } } @@ -833,7 +824,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, case 2: if (!unchanged_attrs(file, stp)) continue; - if (always_checksum && preserve_times + if (always_checksum > 0 && preserve_times && cmp_time(stp->st_mtime, file->modtime)) continue; best_match = j; @@ -1049,7 +1040,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, static int phase = 0; -/* Acts on the_file_list->file's ndx'th item, whose name is fname. If a dir, +/* Acts on cur_flist->file's ndx'th item, whose name is fname. If a dir, * make sure it exists, and has the right permissions/timestamp info. For * all other non-regular files (symlinks, etc.) we create them here. For * regular files that have changed, we try to find a basis file and then @@ -1079,20 +1070,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, if (list_only) return; - if (!fname) { - if (fuzzy_dirlist) { - flist_free(fuzzy_dirlist); - fuzzy_dirlist = NULL; - } - if (missing_below >= 0) { - if (dry_run) - dry_run--; - missing_below = -1; - } - parent_dirname = ""; - return; - } - if (verbose > 2) rprintf(FINFO, "recv_generator(%s,%d)\n", fname, ndx); @@ -1121,10 +1098,18 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, if (dry_run) dry_run--; missing_below = -1; - } else if (!dry_run) + } else if (!dry_run) { + if (S_ISDIR(file->mode)) + file->flags |= FLAG_MISSING_DIR; return; + } } if (dry_run > 1) { + if (fuzzy_dirlist) { + flist_free(fuzzy_dirlist); + fuzzy_dirlist = NULL; + } + parent_dirname = ""; statret = -1; stat_errno = ENOENT; } else { @@ -1157,12 +1142,20 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, stat_errno = errno; } - if (ignore_non_existing && statret == -1 && stat_errno == ENOENT) { + if (ignore_non_existing > 0 && statret == -1 && stat_errno == ENOENT) { if (verbose > 1) { rprintf(FINFO, "not creating new %s \"%s\"\n", S_ISDIR(file->mode) ? "directory" : "file", fname); } + if (S_ISDIR(file->mode)) { + if (missing_below < 0) { + if (dry_run) + dry_run++; + missing_below = F_DEPTH(file); + } + file->flags |= FLAG_MISSING_DIR; + } return; } @@ -1216,13 +1209,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, rsyserr(FERROR, errno, "recv_generator: mkdir %s failed", full_fname(fname)); + rprintf(FERROR, + "*** Skipping any contents from this failed directory ***\n"); + missing_below = F_DEPTH(file); file->flags |= FLAG_MISSING_DIR; - if (ndx+1 < the_file_list->count - && F_DEPTH(the_file_list->files[ndx+1]) > F_DEPTH(file)) { - rprintf(FERROR, - "*** Skipping everything below this failed directory ***\n"); - missing_below = F_DEPTH(file); - } return; } } @@ -1231,9 +1221,16 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, rprintf(code, "%s/\n", fname); if (real_ret != 0 && one_file_system) real_st.st_dev = filesystem_dev; - if (delete_during && f_out != -1 && !phase && dry_run < 2 + if (incremental) { + if (one_file_system) { + uint32 *devp = F_DIRDEV_P(file); + DEV_MAJOR(devp) = major(real_st.st_dev); + DEV_MINOR(devp) = minor(real_st.st_dev); + } + } + else if (delete_during && f_out != -1 && !phase && dry_run < 2 && (file->flags & FLAG_XFER_DIR)) - delete_in_dir(the_file_list, fname, file, &real_st); + delete_in_dir(cur_flist, fname, file, &real_st.st_dev); return; } @@ -1410,7 +1407,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; } - if (max_size && F_LENGTH(file) > max_size) { + if (max_size > 0 && F_LENGTH(file) > max_size) { if (verbose > 1) { if (solo_file) fname = f_name(file, NULL); @@ -1418,7 +1415,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, } return; } - if (min_size && F_LENGTH(file) < min_size) { + if (min_size > 0 && F_LENGTH(file) < min_size) { if (verbose > 1) { if (solo_file) fname = f_name(file, NULL); @@ -1427,13 +1424,13 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; } - if (ignore_existing && statret == 0) { + if (ignore_existing > 0 && statret == 0) { if (verbose > 1) rprintf(FINFO, "%s exists\n", fname); return; } - if (update_only && statret == 0 + if (update_only > 0 && statret == 0 && cmp_time(st.st_mtime, file->modtime) > 0) { if (verbose > 1) rprintf(FINFO, "%s is newer\n", fname); @@ -1504,7 +1501,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; } - if (append_mode && st.st_size > F_LENGTH(file)) + if (append_mode > 0 && st.st_size > F_LENGTH(file)) return; if (fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH) @@ -1527,7 +1524,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; return_with_success: if (!dry_run) - send_msg_int(MSG_SUCCESS, ndx); + send_msg_int(MSG_SUCCESS, ndx + cur_flist->ndx_start); return; } @@ -1545,7 +1542,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, if (fuzzy_dirlist) { int j = flist_find(fuzzy_dirlist, file); if (j >= 0) /* don't use changing file as future fuzzy basis */ - fuzzy_dirlist->files[j]->flags |= FLAG_SENT; + fuzzy_dirlist->files[j]->flags |= FLAG_FILE_SENT; } /* open the file */ @@ -1564,7 +1561,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto notify_others; } - if (inplace && make_backups && fnamecmp_type == FNAMECMP_FNAME) { + if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) { if (!(backupptr = get_backup_name(fname))) { close(fd); return; @@ -1602,14 +1599,16 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, notify_others: if (remove_source_files && !delay_updates && !phase) increment_active_files(ndx, itemizing, code); + if (incremental && !dry_run) + cur_flist->in_progress++; #ifdef SUPPORT_HARD_LINKS if (preserve_hard_links && F_IS_HLINKED(file)) - file->flags |= FLAG_SENT; + file->flags |= FLAG_FILE_SENT; #endif - write_int(f_out, ndx); + write_int(f_out, ndx + cur_flist->ndx_start); if (itemizing) { int iflags = ITEM_TRANSFER; - if (always_checksum) + if (always_checksum > 0) iflags |= ITEM_REPORT_CHECKSUM; if (fnamecmp_type != FNAMECMP_FNAME) iflags |= ITEM_BASIS_TYPE_FOLLOWS; @@ -1649,19 +1648,17 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, close(fd); } -void generate_files(int f_out, struct file_list *flist, char *local_name) +void generate_files(int f_out, char *local_name) { int i; char fbuf[MAXPATHLEN]; int itemizing; enum logcode code; + struct file_list *next_flist; int lull_mod = allowed_lull * 5; int need_retouch_dir_times = preserve_times && !omit_dir_times; int need_retouch_dir_perms = 0; - int save_ignore_existing = ignore_existing; - int save_ignore_non_existing = ignore_non_existing; int save_do_progress = do_progress; - int save_make_backups = make_backups; int dir_tweaking = !(list_only || local_name || dry_run); if (protocol_version >= 29) { @@ -1683,13 +1680,11 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) } solo_file = local_name != NULL; - if (verbose > 2) { - rprintf(FINFO, "generator starting pid=%ld count=%d\n", - (long)getpid(), flist->count); - } + if (verbose > 2) + rprintf(FINFO, "generator starting pid=%ld\n", (long)getpid()); - if (delete_before && !local_name && flist->count > 0) - do_delete_pass(flist); + if (delete_before && !local_name && cur_flist->count > 0) + do_delete_pass(cur_flist); if (delete_during == 2) { deldelay_size = BIGPATHBUFLEN * 4; deldelay_buf = new_array(char, deldelay_size); @@ -1698,7 +1693,7 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) } do_progress = 0; - if (append_mode || whole_file < 0) + if (append_mode > 0 || whole_file < 0) whole_file = 0; if (verbose >= 2) { rprintf(FINFO, "delta-transmission %s\n", @@ -1713,79 +1708,171 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) * notice that and let us know via the redo pipe (or its closing). */ ignore_timeout = 1; - for (i = 0; i < flist->count; i++) { - struct file_struct *file = flist->files[i]; + do { + if (incremental && delete_during && cur_flist->ndx_start) { + struct file_struct *fp = dir_flist->files[cur_flist->parent_ndx]; + if (BITS_SETnUNSET(fp->flags, FLAG_XFER_DIR, FLAG_MISSING_DIR)) { + dev_t dirdev; + if (one_file_system) { + uint32 *devp = F_DIRDEV_P(fp); + dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); + } else + dirdev = MAKEDEV(0, 0); + delete_in_dir(cur_flist, f_name(fp, fbuf), fp, &dirdev); + } + } + for (i = cur_flist->low; i <= cur_flist->high; i++) { + struct file_struct *file = cur_flist->files[i]; - if (!F_IS_ACTIVE(file)) - continue; + if (!F_IS_ACTIVE(file)) + continue; - if (local_name) - strlcpy(fbuf, local_name, sizeof fbuf); - else - f_name(file, fbuf); - recv_generator(fbuf, file, i, itemizing, code, f_out); + if (local_name) + strlcpy(fbuf, local_name, sizeof fbuf); + else + f_name(file, fbuf); + recv_generator(fbuf, file, i, itemizing, code, f_out); - /* We need to ensure that any dirs we create have writeable - * permissions during the time we are putting files within - * them. This is then fixed after the transfer is done. */ + /* We need to ensure that any dirs we create have + * writeable permissions during the time we are putting + * files within them. This is then fixed after the + * transfer is done. */ #ifdef HAVE_CHMOD - if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR) - && dir_tweaking) { - mode_t mode = file->mode | S_IWUSR; /* user write */ - char *fname = local_name ? local_name : fbuf; - if (do_chmod(fname, mode) < 0) { - rsyserr(FERROR, errno, - "failed to modify permissions on %s", - full_fname(fname)); + if (!am_root && S_ISDIR(file->mode) + && !(file->mode & S_IWUSR) && dir_tweaking) { + mode_t mode = file->mode | S_IWUSR; + char *fname = local_name ? local_name : fbuf; + if (do_chmod(fname, mode) < 0) { + rsyserr(FERROR, errno, + "failed to modify permissions on %s", + full_fname(fname)); + } + need_retouch_dir_perms = 1; } - need_retouch_dir_perms = 1; - } #endif #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links) - check_for_finished_hlinks(itemizing, code); + if (preserve_hard_links) + check_for_finished_hlinks(itemizing, code); #endif - if (allowed_lull && !(i % lull_mod)) - maybe_send_keepalive(); - else if (!(i % 200)) - maybe_flush_socket(); - } - recv_generator(NULL, NULL, 0, 0, code, -1); - if (delete_during) - delete_in_dir(NULL, NULL, NULL, NULL); + if (allowed_lull && !(i % lull_mod)) + maybe_send_keepalive(); + else if (!(i % 200)) + maybe_flush_socket(); + } - phase++; - csum_length = SUM_LENGTH; - max_size = min_size = ignore_existing = ignore_non_existing = 0; - update_only = always_checksum = size_only = 0; - ignore_times = 1; - if (append_mode) /* resend w/o append mode */ - append_mode = -1; /* ... but only longer files */ - make_backups = 0; /* avoid a duplicate backup for inplace processing */ + if (!incremental) { + if (delete_during) + delete_in_dir(NULL, NULL, NULL, &dev_zero); + phase++; + if (verbose > 2) { + rprintf(FINFO, "generate_files phase=%d\n", + phase); + } - if (verbose > 2) - rprintf(FINFO, "generate_files phase=%d\n", phase); + write_int(f_out, NDX_DONE); + } - write_int(f_out, NDX_DONE); + csum_length = SUM_LENGTH; + max_size = -max_size; + min_size = -min_size; + ignore_existing = -ignore_existing; + ignore_non_existing = -ignore_non_existing; + update_only = -update_only; + always_checksum = -always_checksum; + size_only = -size_only; + append_mode = -append_mode; + make_backups = -make_backups; /* avoid dup backup w/inplace */ + ignore_times++; + + /* Files can cycle through the system more than once + * to catch initial checksum errors. */ + while (!done_cnt) { + struct file_struct *file; + struct file_list *save_flist; - /* files can cycle through the system more than once - * to catch initial checksum errors */ - while ((i = get_redo_num(itemizing, code)) != -1) { - struct file_struct *file = flist->files[i]; - if (local_name) - strlcpy(fbuf, local_name, sizeof fbuf); - else - f_name(file, fbuf); - recv_generator(fbuf, file, i, itemizing, code, f_out); - } + check_for_finished_hlinks(itemizing, code); - phase++; - ignore_non_existing = save_ignore_non_existing; - ignore_existing = save_ignore_existing; - make_backups = save_make_backups; + if ((i = get_redo_num()) == -1) { + if (incremental) + break; + wait_for_receiver(); + continue; + } + + save_flist = cur_flist; + cur_flist = flist_for_ndx(i); + file = cur_flist->files[i]; + if (local_name) + strlcpy(fbuf, local_name, sizeof fbuf); + else + f_name(file, fbuf); + recv_generator(fbuf, file, i, itemizing, code, f_out); + cur_flist->to_redo--; + cur_flist = save_flist; + } + + csum_length = SHORT_SUM_LENGTH; + max_size = -max_size; + min_size = -min_size; + ignore_existing = -ignore_existing; + ignore_non_existing = -ignore_non_existing; + update_only = -update_only; + always_checksum = -always_checksum; + size_only = -size_only; + append_mode = -append_mode; + make_backups = -make_backups; + ignore_times--; + + if (!incremental) + break; + + while (!cur_flist->next && !flist_eof) + wait_for_receiver(); + next_flist = cur_flist->next; + while (first_flist != next_flist) { + struct file_struct *fp; + if (first_flist->in_progress || first_flist->to_redo) { + if (next_flist) + break; + wait_for_receiver(); + continue; + } + + cur_flist = first_flist; + if (cur_flist->ndx_start != 0) { + fp = dir_flist->files[cur_flist->parent_ndx]; + if (!(fp->flags & FLAG_MISSING_DIR)) { + f_name(fp, fbuf); + if (!(fp->mode & S_IWUSR)) + do_chmod(fbuf, fp->mode); + if (preserve_times && !omit_dir_times) { + set_modtime(fbuf, fp->modtime, + fp->mode); + } + } + } else if (relative_paths && implied_dirs + && preserve_times && !omit_dir_times) { + /* Set mtime on implied dirs */ + for (i = 0; i < cur_flist->count; i++) { + fp = cur_flist->files[i]; + if (!S_ISDIR(fp->mode) + || fp->flags & (FLAG_XFER_DIR|FLAG_MISSING_DIR)) + continue; + f_name(fp, fbuf); + set_modtime(fbuf, fp->modtime, fp->mode); + } + } + flist_free(cur_flist); + + if (!read_batch) + write_int(f_out, NDX_DONE); + } + } while ((cur_flist = next_flist) != NULL); + + phase++; if (verbose > 2) rprintf(FINFO, "generate_files phase=%d\n", phase); @@ -1795,7 +1882,10 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) write_int(f_out, NDX_DONE); /* Read MSG_DONE for the redo phase (and any prior messages). */ - get_redo_num(itemizing, code); + while (done_cnt <= 1) { + check_for_finished_hlinks(itemizing, code); + wait_for_receiver(); + } if (protocol_version >= 29) { phase++; @@ -1804,30 +1894,32 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) if (delay_updates) write_int(f_out, NDX_DONE); /* Read MSG_DONE for delay-updates phase & prior messages. */ - get_redo_num(itemizing, code); + while (done_cnt == 2) + wait_for_receiver(); } do_progress = save_do_progress; if (delete_during == 2) do_delayed_deletions(fbuf); - if (delete_after && !local_name && flist->count > 0) - do_delete_pass(flist); + if (delete_after && !local_name && file_total > 0) + do_delete_pass(cur_flist); - if ((need_retouch_dir_perms || need_retouch_dir_times) && dir_tweaking) { + if ((need_retouch_dir_perms || need_retouch_dir_times) + && dir_tweaking && !incremental) { int j = 0; /* Now we need to fix any directory permissions that were * modified during the transfer and/or re-set any tweaked * modified-time values. */ - for (i = 0; i < flist->count; i++) { - struct file_struct *file = flist->files[i]; + for (i = 0; i < cur_flist->count; i++) { + struct file_struct *file = cur_flist->files[i]; if (!F_IS_ACTIVE(file) || !S_ISDIR(file->mode)) continue; if (!need_retouch_dir_times && file->mode & S_IWUSR) continue; if (file->flags & FLAG_MISSING_DIR) { int missing = F_DEPTH(file); - while (++i < flist->count) { - file = flist->files[i]; + while (++i < cur_flist->count) { + file = cur_flist->files[i]; if (F_DEPTH(file) <= missing) break; } @@ -1842,7 +1934,6 @@ void generate_files(int f_out, struct file_list *flist, char *local_name) maybe_flush_socket(); } } - recv_generator(NULL, NULL, 0, 0, code, -1); if (max_delete >= 0 && deletion_count > max_delete) { rprintf(FINFO, diff --git a/main.c b/main.c index 76692e1b..2d2b697f 100644 --- a/main.c +++ b/main.c @@ -34,6 +34,7 @@ extern int am_server; extern int am_sender; extern int am_generator; extern int am_daemon; +extern int incremental; extern int blocking_io; extern int remove_source_files; extern int daemon_over_rsh; @@ -47,6 +48,7 @@ extern int copy_dirlinks; extern int keep_dirlinks; extern int preserve_hard_links; extern int protocol_version; +extern int file_total; extern int recurse; extern int relative_paths; extern int sanitize_paths; @@ -58,7 +60,6 @@ extern int whole_file; extern int read_batch; extern int write_batch; extern int batch_fd; -extern int batch_gen_fd; extern int filesfrom_fd; extern pid_t cleanup_child_pid; extern struct stats stats; @@ -75,7 +76,7 @@ extern struct filter_list_struct server_filter_list; int local_server = 0; int new_root_dir = 0; mode_t orig_umask = 0; -struct file_list *the_file_list; +int batch_gen_fd = -1; /* There's probably never more than at most 2 outstanding child processes, * but set it higher, just in case. */ @@ -476,7 +477,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path) if (verbose > 2) { rprintf(FINFO, "get_local_name count=%d %s\n", - flist->count, NS(dest_path)); + file_total, NS(dest_path)); } if (!dest_path || list_only) @@ -493,14 +494,13 @@ static char *get_local_name(struct file_list *flist, char *dest_path) } return NULL; } - if (flist->count > 1) { + if (file_total > 1) { rprintf(FERROR, "ERROR: destination must be a directory when" " copying more than 1 file\n"); exit_cleanup(RERR_FILESELECT); } - /* Caution: flist->count could be 0! */ - if (flist->count == 1 && S_ISDIR(flist->files[0]->mode)) { + if (file_total == 1 && S_ISDIR(flist->files[0]->mode)) { rprintf(FERROR, "ERROR: cannot overwrite non-directory" " with a directory\n"); @@ -518,7 +518,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path) /* If we need a destination directory because the transfer is not * of a single non-directory or the user has requested one via a * destination path ending in a slash, create one and use mode 1. */ - if (flist->count > 1 || (cp && !cp[1])) { + if (file_total > 1 || (cp && !cp[1])) { /* Lop off the final slash (if any). */ if (cp && !cp[1]) *cp = '\0'; @@ -611,17 +611,15 @@ static void fix_basis_dirs(void) /* This is only called by the sender. */ static void read_final_goodbye(int f_in, int f_out) { - int i; + int i, iflags, xlen; + uchar fnamecmp_type; + char xname[MAXPATHLEN]; if (protocol_version < 29) i = read_int(f_in); else { - while ((i = read_int(f_in)) == the_file_list->count - && read_shortint(f_in) == ITEM_IS_NEW) { - /* Forward the keep-alive (no-op) to the receiver. */ - write_int(f_out, the_file_list->count); - write_shortint(f_out, ITEM_IS_NEW); - } + i = read_ndx_and_attrs(f_in, f_out, &iflags, + &fnamecmp_type, xname, &xlen); } if (i != NDX_DONE) { @@ -673,12 +671,10 @@ static void do_server_sender(int f_in, int f_out, int argc, char *argv[]) flist = send_file_list(f_out,argc,argv); if (!flist || flist->count == 0) exit_cleanup(0); - the_file_list = flist; - io_start_buffering_in(); - io_start_buffering_out(); + io_start_buffering_in(f_in); - send_files(flist,f_out,f_in); + send_files(f_in, f_out); io_flush(FULL_FLUSH); handle_stats(f_out); if (protocol_version >= 24) @@ -688,7 +684,7 @@ static void do_server_sender(int f_in, int f_out, int argc, char *argv[]) } -static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) +static int do_recv(int f_in, int f_out, char *local_name) { int pid; int exit_code = 0; @@ -721,12 +717,13 @@ static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) close(f_out); /* we can't let two processes write to the socket at one time */ - close_multiplexing_out(); + io_end_multiplex_out(); /* set place to send errors */ set_msg_fd_out(error_pipe[1]); + io_start_buffering_out(error_pipe[1]); - recv_files(f_in, flist, local_name); + recv_files(f_in, local_name); io_flush(FULL_FLUSH); handle_stats(f_in); @@ -736,11 +733,15 @@ static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) /* Handle any keep-alive packets from the post-processing work * that the generator does. */ if (protocol_version >= 29) { + int iflags, xlen; + uchar fnamecmp_type; + char xname[MAXPATHLEN]; + kluge_around_eof = -1; /* This should only get stopped via a USR2 signal. */ - while (read_int(f_in) == flist->count - && read_shortint(f_in) == ITEM_IS_NEW) {} + read_ndx_and_attrs(f_in, -1, &iflags, &fnamecmp_type, + xname, &xlen); rprintf(FERROR, "Invalid packet at end of run [%s]\n", who_am_i()); @@ -755,7 +756,8 @@ static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) } am_generator = 1; - close_multiplexing_in(); + + io_end_multiplex_in(); if (write_batch && !am_server) stop_write_batch(); @@ -763,11 +765,12 @@ static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) if (f_in != f_out) close(f_in); - io_start_buffering_out(); + io_start_buffering_out(f_out); set_msg_fd_in(error_pipe[0]); + io_start_buffering_in(error_pipe[0]); - generate_files(f_out, flist, local_name); + generate_files(f_out, local_name); handle_stats(-1); io_flush(FULL_FLUSH); @@ -783,7 +786,7 @@ static int do_recv(int f_in,int f_out,struct file_list *flist,char *local_name) return exit_code; } -static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) +static void do_server_recv(int f_in, int f_out, int argc, char *argv[]) { int exit_code; struct file_list *flist; @@ -819,7 +822,10 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) } } - io_start_buffering_in(); + if (protocol_version >= 30) + io_start_multiplex_in(); + else + io_start_buffering_in(f_in); recv_filter_list(f_in); if (filesfrom_fd >= 0) { @@ -833,12 +839,13 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) } flist = recv_file_list(f_in); - verbose = save_verbose; if (!flist) { rprintf(FERROR,"server_recv: recv_file_list error\n"); exit_cleanup(RERR_FILESELECT); } - the_file_list = flist; + if (incremental && file_total == 1) + recv_additional_file_list(f_in); + verbose = save_verbose; if (argc > 0) local_name = get_local_name(flist,argv[0]); @@ -873,7 +880,7 @@ static void do_server_recv(int f_in, int f_out, int argc,char *argv[]) } } - exit_code = do_recv(f_in,f_out,flist,local_name); + exit_code = do_recv(f_in, f_out, local_name); exit_cleanup(exit_code); } @@ -933,9 +940,6 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) setup_iconv(); #endif - if (protocol_version >= 23 && !read_batch) - io_start_multiplex_in(); - /* We set our stderr file handle to blocking because ssh might have * set it to non-blocking. This can be particularly troublesome if * stderr is a clone of stdout, because ssh would have set our stdout @@ -948,7 +952,10 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) if (am_sender) { keep_dirlinks = 0; /* Must be disabled on the sender. */ - io_start_buffering_out(); + if (protocol_version >= 30) + io_start_multiplex_out(); + else + io_start_buffering_out(f_out); if (!filesfrom_host) set_msg_fd_in(f_in); send_filter_list(f_out); @@ -961,10 +968,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) set_msg_fd_in(-1); if (verbose > 3) rprintf(FINFO,"file list sent\n"); - the_file_list = flist; + + if (protocol_version >= 23) + io_start_multiplex_in(); io_flush(NORMAL_FLUSH); - send_files(flist,f_out,f_in); + send_files(f_in, f_out); io_flush(FULL_FLUSH); handle_stats(-1); if (protocol_version >= 24) @@ -980,8 +989,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) exit_cleanup(exit_code); } - if (need_messages_from_generator && !read_batch) - io_start_multiplex_out(); + if (!read_batch) { + if (protocol_version >= 23) + io_start_multiplex_in(); + if (need_messages_from_generator) + io_start_multiplex_out(); + } if (argc == 0) list_only |= 1; @@ -996,14 +1009,15 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) if (write_batch && !am_server) start_write_batch(f_in); flist = recv_file_list(f_in); - the_file_list = flist; + if (incremental && file_total == 1) + recv_additional_file_list(f_in); if (flist && flist->count > 0) { local_name = get_local_name(flist, argv[0]); fix_basis_dirs(); - exit_code2 = do_recv(f_in, f_out, flist, local_name); + exit_code2 = do_recv(f_in, f_out, local_name); } else { handle_stats(-1); output_summary(); diff --git a/receiver.c b/receiver.c index 7bf14c7d..26ed3245 100644 --- a/receiver.c +++ b/receiver.c @@ -26,6 +26,7 @@ extern int verbose; extern int do_xfers; extern int am_server; extern int do_progress; +extern int incremental; extern int log_before_transfer; extern int stdout_format_has_i; extern int logfile_format_has_i; @@ -52,7 +53,7 @@ extern char *stdout_format; extern char *tmpdir; extern char *partial_dir; extern char *basis_dir[]; -extern struct file_list *the_file_list; +extern struct file_list *cur_flist, *first_flist; extern struct filter_list_struct server_filter_list; static struct bitbag *delayed_bits = NULL; @@ -60,7 +61,6 @@ static int phase = 0; /* We're either updating the basis file or an identical copy: */ static int updating_basis; - /* * get_tmpname() - create a tmp filename for a given filename * @@ -151,7 +151,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, sum_init(checksum_seed); - if (append_mode) { + if (append_mode > 0) { OFF_T j; sum.flength = (OFF_T)sum.count * sum.blength; if (sum.remainder) @@ -277,16 +277,16 @@ static void discard_receive_data(int f_in, OFF_T length) receive_data(f_in, NULL, -1, 0, NULL, -1, length); } -static void handle_delayed_updates(struct file_list *flist, char *local_name) +static void handle_delayed_updates(char *local_name) { char *fname, *partialptr; int ndx; for (ndx = -1; (ndx = bitbag_next_bit(delayed_bits, ndx)) >= 0; ) { - struct file_struct *file = flist->files[ndx]; + struct file_struct *file = cur_flist->files[ndx]; fname = local_name ? local_name : f_name(file, NULL); if ((partialptr = partial_dir_fname(fname)) != NULL) { - if (make_backups && !make_backup(fname)) + if (make_backups > 0 && !make_backup(fname)) continue; if (verbose > 2) { rprintf(FINFO, "renaming %s to %s\n", @@ -308,28 +308,31 @@ static void handle_delayed_updates(struct file_list *flist, char *local_name) } } -static int get_next_gen_ndx(int batch_gen_fd, int next_gen_ndx, int desired_ndx) +static int get_next_gen_ndx(int fd, int next_gen_ndx, int desired_ndx) { while (next_gen_ndx < desired_ndx) { if (next_gen_ndx >= 0) { rprintf(FINFO, "(No batched update for%s \"%s\")\n", phase ? " resend of" : "", - f_name(the_file_list->files[next_gen_ndx], NULL)); + f_name(cur_flist->files[next_gen_ndx], NULL)); + } + next_gen_ndx = read_int(fd); + if (next_gen_ndx == -1) { + if (incremental) + next_gen_ndx = first_flist->prev->count + first_flist->prev->ndx_start; + else + next_gen_ndx = cur_flist->count; } - next_gen_ndx = read_int(batch_gen_fd); - if (next_gen_ndx == -1) - next_gen_ndx = the_file_list->count; } return next_gen_ndx; } - /** * main routine for receiver process. * * Receiver process runs on the same host as the generator process. */ -int recv_files(int f_in, struct file_list *flist, char *local_name) +int recv_files(int f_in, char *local_name) { int next_gen_ndx = -1; int fd1,fd2; @@ -343,53 +346,49 @@ int recv_files(int f_in, struct file_list *flist, char *local_name) uchar fnamecmp_type; struct file_struct *file; struct stats initial_stats; - int save_make_backups = make_backups; int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i; enum logcode log_code = log_before_transfer ? FLOG : FINFO; int max_phase = protocol_version >= 29 ? 2 : 1; int ndx, recv_ok; if (verbose > 2) - rprintf(FINFO, "recv_files(%d) starting\n", flist->count); + rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->count); if (delay_updates) - delayed_bits = bitbag_create(flist->count); + delayed_bits = bitbag_create(cur_flist->count + 1); updating_basis = inplace; while (1) { cleanup_disable(); - ndx = read_int(f_in); + /* This call also sets cur_flist. */ + ndx = read_ndx_and_attrs(f_in, -1, &iflags, + &fnamecmp_type, xname, &xlen); if (ndx == NDX_DONE) { - if (read_batch) { - get_next_gen_ndx(batch_gen_fd, next_gen_ndx, - flist->count); + if (incremental && first_flist) { + flist_free(first_flist); + if (first_flist) + continue; + } + if (read_batch && cur_flist) { + int high = incremental + ? first_flist->prev->count + first_flist->prev->ndx_start + : cur_flist->count; + get_next_gen_ndx(batch_gen_fd, next_gen_ndx, high); next_gen_ndx = -1; } if (++phase > max_phase) break; - csum_length = SUM_LENGTH; if (verbose > 2) rprintf(FINFO, "recv_files phase=%d\n", phase); if (phase == 2 && delay_updates) - handle_delayed_updates(flist, local_name); + handle_delayed_updates(local_name); send_msg(MSG_DONE, "", 0); - if (keep_partial && !partial_dir) - make_backups = 0; /* prevents double backup */ - if (append_mode) { - append_mode = 0; - sparse_files = 0; - } continue; } - iflags = read_item_attrs(f_in, -1, ndx, &fnamecmp_type, - xname, &xlen); - if (iflags == ITEM_IS_NEW) /* no-op packet */ - continue; - - file = flist->files[ndx]; + file = cur_flist->files[ndx - cur_flist->ndx_start]; fname = local_name ? local_name : f_name(file, fbuf); if (verbose > 2) @@ -406,6 +405,24 @@ int recv_files(int f_in, struct file_list *flist, char *local_name) exit_cleanup(RERR_PROTOCOL); } + if (file->flags & FLAG_FILE_SENT) { + if (csum_length == SHORT_SUM_LENGTH) { + if (keep_partial && !partial_dir) + make_backups = -make_backups; /* prevents double backup */ + append_mode = -append_mode; + sparse_files = -sparse_files; + csum_length = SUM_LENGTH; + } + } else { + if (csum_length != SHORT_SUM_LENGTH) { + if (keep_partial && !partial_dir) + make_backups = -make_backups; + append_mode = -append_mode; + sparse_files = -sparse_files; + csum_length = SHORT_SUM_LENGTH; + } + } + stats.current_file_index = ndx; stats.num_transferred_files++; stats.total_transferred_size += F_LENGTH(file); @@ -483,7 +500,7 @@ int recv_files(int f_in, struct file_list *flist, char *local_name) } else { /* Reminder: --inplace && --partial-dir are never * enabled at the same time. */ - if (inplace && make_backups) { + if (inplace && make_backups > 0) { if (!(fnamecmp = get_backup_name(fname))) fnamecmp = fname; } else if (partial_dir && partialptr) @@ -646,7 +663,7 @@ int recv_files(int f_in, struct file_list *flist, char *local_name) cleanup_disable(); if (recv_ok > 0) { - if (remove_source_files + if (remove_source_files || incremental || (preserve_hard_links && F_IS_HLINKED(file))) send_msg_int(MSG_SUCCESS, ndx); } else if (!recv_ok) { @@ -670,14 +687,17 @@ int recv_files(int f_in, struct file_list *flist, char *local_name) "%s: %s failed verification -- update %s%s.\n", errstr, fname, keptstr, redostr); } - if (!phase) + if (!phase || incremental) { send_msg_int(MSG_REDO, ndx); + file->flags |= FLAG_FILE_SENT; + } } } - make_backups = save_make_backups; + if (make_backups < 0) + make_backups = -make_backups; if (phase == 2 && delay_updates) /* for protocol_version < 29 */ - handle_delayed_updates(flist, local_name); + handle_delayed_updates(local_name); if (verbose > 2) rprintf(FINFO,"recv_files finished\n"); diff --git a/rsync.c b/rsync.c index 0832be47..a789ebd8 100644 --- a/rsync.c +++ b/rsync.c @@ -45,12 +45,14 @@ extern int allow_8bit_chars; extern int protocol_version; extern int preserve_uid; extern int preserve_gid; +extern int incremental; extern int inplace; +extern int flist_eof; extern int keep_dirlinks; extern int make_backups; extern mode_t orig_umask; extern struct stats stats; -extern struct file_list *the_file_list; +extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct chmod_mode_struct *daemon_chmod_modes; #if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H @@ -93,26 +95,72 @@ void setup_iconv() /* This is used by sender.c with a valid f_out, and by receive.c with * f_out = -1. */ -int read_item_attrs(int f_in, int f_out, int ndx, uchar *type_ptr, - char *buf, int *len_ptr) +int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, + uchar *type_ptr, char *buf, int *len_ptr) { - int len; + int len, iflags = 0; + struct file_list *flist; uchar fnamecmp_type = FNAMECMP_FNAME; - int iflags = protocol_version >= 29 ? read_shortint(f_in) + int verbose_save, ndx; + + read_loop: + while (1) { + ndx = read_int(f_in); + + if (ndx >= 0) + break; + if (ndx == NDX_DONE) + return ndx; + if (!incremental || am_sender) + goto invalid_ndx; + if (ndx == NDX_FLIST_EOF) { + flist_eof = 1; + send_msg(MSG_FLIST_EOF, "", 0); + continue; + } + ndx = NDX_FLIST_OFFSET - ndx; + if (ndx < 0 || ndx >= dir_flist->count) { + ndx = NDX_FLIST_OFFSET - ndx; + rprintf(FERROR, + "Invalid dir index: %d (%d - %d)\n", + ndx, NDX_FLIST_OFFSET, + NDX_FLIST_OFFSET - dir_flist->count); + exit_cleanup(RERR_PROTOCOL); + } + verbose_save = verbose; + verbose = 0; /* TODO allow verbose messages? */ + + /* Send everything read from f_in to msg_fd_out. */ + send_msg_int(MSG_FLIST, ndx); + start_flist_forward(f_in); + flist = recv_file_list(f_in); + flist->parent_ndx = ndx; + stop_flist_forward(); + + verbose = verbose_save; + } + + iflags = protocol_version >= 29 ? read_shortint(f_in) : ITEM_TRANSFER | ITEM_MISSING_DATA; - /* Handle the new keep-alive (no-op) packet. */ - if (ndx == the_file_list->count && iflags == ITEM_IS_NEW) - ; - else if (ndx < 0 || ndx >= the_file_list->count) { - rprintf(FERROR, "Invalid file index: %d (count=%d) [%s]\n", - ndx, the_file_list->count, who_am_i()); - exit_cleanup(RERR_PROTOCOL); - } else if (iflags == ITEM_IS_NEW) { - rprintf(FERROR, "Invalid itemized flag word: %x [%s]\n", - iflags, who_am_i()); + /* Honor the old-style keep-alive indicator. */ + if (protocol_version < 30 + && ndx == cur_flist->count && iflags == ITEM_IS_NEW) { + if (am_sender) + maybe_send_keepalive(); + goto read_loop; + } + + if (!(flist = flist_for_ndx(ndx))) { + invalid_ndx: + rprintf(FERROR, + "Invalid file index: %d (%d - %d) with iflags %x [%s]\n", + ndx, first_flist->ndx_start + first_flist->ndx_start, + first_flist->prev->ndx_start + first_flist->ndx_start + + first_flist->prev->count - 1, iflags, who_am_i()); exit_cleanup(RERR_PROTOCOL); } + cur_flist = flist; if (iflags & ITEM_BASIS_TYPE_FOLLOWS) fnamecmp_type = read_byte(f_in); @@ -128,7 +176,8 @@ int read_item_attrs(int f_in, int f_out, int ndx, uchar *type_ptr, *len_ptr = len; if (iflags & ITEM_TRANSFER) { - if (!S_ISREG(the_file_list->files[ndx]->mode)) { + int i = ndx - cur_flist->ndx_start; + if (!S_ISREG(cur_flist->files[i]->mode)) { rprintf(FERROR, "received request to transfer non-regular file: %d [%s]\n", ndx, who_am_i()); @@ -139,7 +188,8 @@ int read_item_attrs(int f_in, int f_out, int ndx, uchar *type_ptr, fnamecmp_type, buf, len); } - return iflags; + *iflag_ptr = iflags; + return ndx; } /* @@ -314,7 +364,7 @@ void finish_transfer(char *fname, char *fnametmp, char *partialptr, goto do_set_file_attrs; } - if (make_backups && overwriting_basis && !make_backup(fname)) + if (make_backups > 0 && overwriting_basis && !make_backup(fname)) return; /* Change permissions before putting the file into place. */ @@ -354,6 +404,25 @@ void finish_transfer(char *fname, char *fnametmp, char *partialptr, } } +struct file_list *flist_for_ndx(int ndx) +{ + struct file_list *flist = cur_flist; + + if (!flist) + return NULL; + + while (ndx < flist->ndx_start) { + if (flist == first_flist) + return NULL; + flist = flist->prev; + } + while (ndx >= flist->ndx_start + flist->count) { + if (!(flist = flist->next)) + return NULL; + } + return flist; +} + const char *who_am_i(void) { if (am_starting_up) diff --git a/sender.c b/sender.c index 3808cda8..0175c9e4 100644 --- a/sender.c +++ b/sender.c @@ -26,6 +26,7 @@ extern int verbose; extern int do_xfers; extern int am_server; extern int am_daemon; +extern int incremental; extern int log_before_transfer; extern int stdout_format_has_i; extern int logfile_format_has_i; @@ -42,10 +43,9 @@ extern int inplace; extern int batch_fd; extern int write_batch; extern struct stats stats; -extern struct file_list *the_file_list; +extern struct file_list *cur_flist, *first_flist; extern char *stdout_format; - /** * @file * @@ -76,7 +76,7 @@ static struct sum_struct *receive_sums(int f) (double)s->count, (long)s->blength, (long)s->remainder); } - if (append_mode) { + if (append_mode > 0) { s->flength = (OFF_T)s->count * s->blength; if (s->remainder) s->flength -= s->blength - s->remainder; @@ -122,25 +122,28 @@ void successful_send(int ndx) { char fname[MAXPATHLEN]; struct file_struct *file; - unsigned int offset; + struct file_list *flist; - if (ndx < 0 || ndx >= the_file_list->count) + if (!remove_source_files) return; - file = the_file_list->files[ndx]; - if (F_ROOTDIR(file)) { - offset = stringjoin(fname, sizeof fname, - F_ROOTDIR(file), "/", NULL); - } else - offset = 0; - f_name(file, fname + offset); - if (remove_source_files) { - if (do_unlink(fname) == 0) { - if (verbose > 1) - rprintf(FINFO, "sender removed %s\n", fname + offset); - } else - rsyserr(FERROR, errno, "sender failed to remove %s", fname + offset); + if (!(flist = flist_for_ndx(ndx))) { + rprintf(FERROR, + "INTERNAL ERROR: unable to find flist for item %d\n", + ndx); + return; } + + file = flist->files[ndx - flist->ndx_start]; + if (!push_flist_dir(F_ROOTDIR(file), -1)) + return; + f_name(file, fname); + + if (do_unlink(fname) == 0) { + if (verbose > 1) + rprintf(FINFO, "sender removed %s\n", fname); + } else + rsyserr(FERROR, errno, "sender failed to remove %s", fname); } void write_ndx_and_attrs(int f_out, int ndx, int iflags, @@ -156,63 +159,63 @@ void write_ndx_and_attrs(int f_out, int ndx, int iflags, write_vstring(f_out, buf, len); } -void send_files(struct file_list *flist, int f_out, int f_in) +void send_files(int f_in, int f_out) { int fd = -1; struct sum_struct *s; struct map_struct *mbuf = NULL; STRUCT_STAT st; - char *fname2, fname[MAXPATHLEN]; - char xname[MAXPATHLEN]; + char fname[MAXPATHLEN], xname[MAXPATHLEN]; + const char *path, *slash; uchar fnamecmp_type; int iflags, xlen; struct file_struct *file; int phase = 0, max_phase = protocol_version >= 29 ? 2 : 1; struct stats initial_stats; - int save_make_backups = make_backups; int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i; enum logcode log_code = log_before_transfer ? FLOG : FINFO; int f_xfer = write_batch < 0 ? batch_fd : f_out; - int i, j; + int ndx, j; if (verbose > 2) rprintf(FINFO, "send_files starting\n"); while (1) { - unsigned int offset; - - i = read_int(f_in); - if (i == NDX_DONE) { + if (incremental) + send_extra_file_list(f_out, 1000); + + /* This call also sets cur_flist. */ + ndx = read_ndx_and_attrs(f_in, f_out, &iflags, + &fnamecmp_type, xname, &xlen); + if (ndx == NDX_DONE) { + if (incremental && first_flist) { + flist_free(first_flist); + if (first_flist) { + write_int(f_out, NDX_DONE); + continue; + } + } if (++phase > max_phase) break; - csum_length = SUM_LENGTH; if (verbose > 2) rprintf(FINFO, "send_files phase=%d\n", phase); write_int(f_out, NDX_DONE); - /* For inplace: redo phase turns off the backup - * flag so that we do a regular inplace send. */ - make_backups = 0; - append_mode = 0; continue; } - iflags = read_item_attrs(f_in, f_out, i, &fnamecmp_type, - xname, &xlen); - if (iflags == ITEM_IS_NEW) /* no-op packet */ - continue; - - file = flist->files[i]; + file = cur_flist->files[ndx - cur_flist->ndx_start]; if (F_ROOTDIR(file)) { - /* N.B. We're sure that this fits, so offset is OK. */ - offset = strlcpy(fname, F_ROOTDIR(file), sizeof fname); - if (!offset || fname[offset-1] != '/') - fname[offset++] = '/'; - } else - offset = 0; - fname2 = f_name(file, fname + offset); + path = F_ROOTDIR(file); + slash = "/"; + } else { + path = slash = ""; + } + if (!push_flist_dir(F_ROOTDIR(file), -1)) + continue; + f_name(file, fname); if (verbose > 2) - rprintf(FINFO, "send_files(%d, %s)\n", i, fname); + rprintf(FINFO, "send_files(%d, %s%s%s)\n", ndx, path,slash,fname); if (!(iflags & ITEM_TRANSFER)) { maybe_log_item(file, iflags, itemizing, xname); @@ -225,16 +228,32 @@ void send_files(struct file_list *flist, int f_out, int f_in) exit_cleanup(RERR_PROTOCOL); } + if (file->flags & FLAG_FILE_SENT) { + if (csum_length == SHORT_SUM_LENGTH) { + /* For inplace: redo phase turns off the backup + * flag so that we do a regular inplace send. */ + make_backups = -make_backups; + append_mode = -append_mode; + csum_length = SUM_LENGTH; + } + } else { + if (csum_length != SHORT_SUM_LENGTH) { + make_backups = -make_backups; + append_mode = -append_mode; + csum_length = SHORT_SUM_LENGTH; + } + } + updating_basis_file = inplace && (protocol_version >= 29 - ? fnamecmp_type == FNAMECMP_FNAME : !make_backups); + ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0); - stats.current_file_index = i; + stats.current_file_index = ndx; stats.num_transferred_files++; stats.total_transferred_size += F_LENGTH(file); if (!do_xfers) { /* log the transfer */ log_item(FCLIENT, file, &stats, iflags, NULL); - write_ndx_and_attrs(f_out, i, iflags, fnamecmp_type, + write_ndx_and_attrs(f_out, ndx, iflags, fnamecmp_type, xname, xlen); continue; } @@ -244,7 +263,7 @@ void send_files(struct file_list *flist, int f_out, int f_in) if (!(s = receive_sums(f_in))) { io_error |= IOERR_GENERAL; rprintf(FERROR, "receive_sums failed\n"); - return; + exit_cleanup(RERR_PROTOCOL); } fd = do_open(fname, O_RDONLY, 0); @@ -263,6 +282,8 @@ void send_files(struct file_list *flist, int f_out, int f_in) full_fname(fname)); } free_sums(s); + if (protocol_version >= 30) + send_msg_int(MSG_NO_SEND, ndx); continue; } @@ -272,7 +293,7 @@ void send_files(struct file_list *flist, int f_out, int f_in) rsyserr(FERROR, errno, "fstat failed"); free_sums(s); close(fd); - return; + exit_cleanup(RERR_PROTOCOL); } if (st.st_size) { @@ -282,21 +303,21 @@ void send_files(struct file_list *flist, int f_out, int f_in) mbuf = NULL; if (verbose > 2) { - rprintf(FINFO, "send_files mapped %s of size %.0f\n", - fname, (double)st.st_size); + rprintf(FINFO, "send_files mapped %s%s%s of size %.0f\n", + path,slash,fname, (double)st.st_size); } - write_ndx_and_attrs(f_out, i, iflags, fnamecmp_type, + write_ndx_and_attrs(f_out, ndx, iflags, fnamecmp_type, xname, xlen); write_sum_head(f_xfer, s); if (verbose > 2) - rprintf(FINFO, "calling match_sums %s\n", fname); + rprintf(FINFO, "calling match_sums %s%s%s\n", path,slash,fname); if (log_before_transfer) log_item(FCLIENT, file, &initial_stats, iflags, NULL); else if (!am_server && verbose && do_progress) - rprintf(FCLIENT, "%s\n", fname2); + rprintf(FCLIENT, "%s\n", fname); set_compression(fname); @@ -320,12 +341,13 @@ void send_files(struct file_list *flist, int f_out, int f_in) free_sums(s); if (verbose > 2) - rprintf(FINFO, "sender finished %s\n", fname); + rprintf(FINFO, "sender finished %s%s%s\n", path,slash,fname); /* Flag that we actually sent this entry. */ - file->flags |= FLAG_SENT; + file->flags |= FLAG_FILE_SENT; } - make_backups = save_make_backups; + if (make_backups < 0) + make_backups = -make_backups; if (verbose > 2) rprintf(FINFO, "send files finished\n"); -- 2.34.1