From e366e5303fa25e3d007691109293901ad7a7b6f6 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Fri, 26 Sep 2008 21:55:44 -0700 Subject: [PATCH] Enhanced the --stats output: - Mention how many files were created (protocol >= 29). - Mention how many files were deleted (new in protocol 31). - Follow the file-count, created-count, and deleted-count with a break-out list of each count by type. --- NEWS | 10 +++++++++ flist.c | 59 ++++++++++++++++++++++++++++++++++----------------- generator.c | 34 +++++++++++++++++++++-------- main.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++--- progress.c | 2 +- receiver.c | 24 ++++++++++++++++++++- rsync.h | 8 ++++--- sender.c | 21 +++++++++++++++++- 8 files changed, 182 insertions(+), 37 deletions(-) diff --git a/NEWS b/NEWS index d34f5555..d3aeb029 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,16 @@ Changes since 3.0.4: control over what is output. Added an extra type of --progress output using --info=progress2. + - Output numbers in 3-digit groups by default (e.g. 1,234,567). (See the + --human-readable option for a way to turn it off.) + + - Added a "T" (terabyte) category to the --human-readable size suffixes. + + - Enhanced the --stats output: 1) to mention how many files were created + (protocol >= 28), 2) to mention how many files were deleted (new for + protocol 31), and 3) to follow the file-count, created-count, and + deleted-count with a break-out list of each count by type. + - Added the "%C" escape to the log-output handling, which will output the MD5 checksum of any transferred file, or all files if --checksum was specified (when protocol 30 or above is in effect). diff --git a/flist.c b/flist.c index 766f1b37..cb667aa1 100644 --- a/flist.c +++ b/flist.c @@ -124,7 +124,6 @@ static char tmp_sum[MAX_DIGEST_LEN]; static char empty_sum[MAX_DIGEST_LEN]; static int flist_count_offset; /* for --delete --progress */ -static int dir_count = 0; static void flist_sort_and_clean(struct file_list *flist, int strip_root); static void output_flist(struct file_list *flist); @@ -412,17 +411,29 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, int l1, l2; int xflags; - /* Initialize starting value of xflags. */ - if (protocol_version >= 30 && S_ISDIR(file->mode)) { - dir_count++; - if (file->flags & FLAG_CONTENT_DIR) - xflags = file->flags & FLAG_TOP_DIR; - else if (file->flags & FLAG_IMPLIED_DIR) - xflags = XMIT_TOP_DIR | XMIT_NO_CONTENT_DIR; + /* Initialize starting value of xflags and adjust counts. */ + if (S_ISREG(file->mode)) + xflags = 0; + else if (S_ISDIR(file->mode)) { + stats.num_dirs++; + if (protocol_version >= 30) { + if (file->flags & FLAG_CONTENT_DIR) + xflags = file->flags & FLAG_TOP_DIR; + else if (file->flags & FLAG_IMPLIED_DIR) + xflags = XMIT_TOP_DIR | XMIT_NO_CONTENT_DIR; + else + xflags = XMIT_NO_CONTENT_DIR; + } else + xflags = file->flags & FLAG_TOP_DIR; /* FLAG_TOP_DIR == XMIT_TOP_DIR */ + } else { + if (S_ISLNK(file->mode)) + stats.num_symlinks++; + else if (IS_DEVICE(file->mode)) + stats.num_devices++; else - xflags = XMIT_NO_CONTENT_DIR; - } else - xflags = file->flags & FLAG_TOP_DIR; /* FLAG_TOP_DIR == XMIT_TOP_DIR */ + stats.num_specials++; + xflags = 0; + } if (file->mode == mode) xflags |= XMIT_SAME_MODE; @@ -1350,7 +1361,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(F_SUM(file), tmp_sum, checksum_len); if (unsort_ndx) - F_NDX(file) = dir_count; + F_NDX(file) = stats.num_dirs; return file; } @@ -1884,7 +1895,7 @@ void send_extra_file_list(int f, int at_least) old_cnt += flist->used; while (file_total - old_cnt < at_least) { struct file_struct *file = dir_flist->sorted[send_dir_ndx]; - int dir_ndx, dstart = dir_count; + int dir_ndx, dstart = stats.num_dirs; const char *pathname = F_PATHNAME(file); int32 *dp; @@ -1934,7 +1945,7 @@ void send_extra_file_list(int f, int at_least) flist_sort_and_clean(flist, 0); - add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart); + add_dirs_to_tree(send_dir_ndx, flist, stats.num_dirs - dstart); flist_done_allocating(flist); file_total += flist->used; @@ -2280,7 +2291,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) if (inc_recurse) { send_dir_depth = 1; - add_dirs_to_tree(-1, flist, dir_count); + add_dirs_to_tree(-1, flist, stats.num_dirs); if (!file_total || strcmp(flist->sorted[flist->low]->basename, ".") != 0) flist->parent_ndx = -1; flist_done_allocating(flist); @@ -2352,10 +2363,20 @@ struct file_list *recv_file_list(int f) flist_expand(flist, 1); file = recv_file_entry(flist, flags, f); - if (inc_recurse && S_ISDIR(file->mode)) { - flist_expand(dir_flist, 1); - dir_flist->files[dir_flist->used++] = file; - } + if (S_ISREG(file->mode)) { + /* Already counted */ + } else if (S_ISDIR(file->mode)) { + if (inc_recurse) { + flist_expand(dir_flist, 1); + dir_flist->files[dir_flist->used++] = file; + } + stats.num_dirs++; + } else if (S_ISLNK(file->mode)) + stats.num_symlinks++; + else if (IS_DEVICE(file->mode)) + stats.num_symlinks++; + else + stats.num_specials++; flist->files[flist->used++] = file; diff --git a/generator.c b/generator.c index 4bc3f67a..6a0d5271 100644 --- a/generator.c +++ b/generator.c @@ -104,7 +104,7 @@ int non_perishable_cnt = 0; int maybe_ATTRS_REPORT = 0; static dev_t dev_zero; -static int deletion_count = 0; /* used to implement --max-delete */ +static int skipped_deletes = 0; static int deldelay_size = 0, deldelay_cnt = 0; static char *deldelay_buf = NULL; static int deldelay_fd = -1; @@ -187,8 +187,10 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) /* OK: try to delete the directory. */ } - if (!(flags & DEL_MAKE_ROOM) && max_delete >= 0 && ++deletion_count > max_delete) + if (!(flags & DEL_MAKE_ROOM) && max_delete >= 0 && stats.deleted_files >= max_delete) { + skipped_deletes++; return DR_AT_LIMIT; + } if (S_ISDIR(mode)) { what = "rmdir"; @@ -202,8 +204,22 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) } if (ok) { - if (!(flags & DEL_MAKE_ROOM)) + if (!(flags & DEL_MAKE_ROOM)) { log_delete(fbuf, mode); + stats.deleted_files++; + if (S_ISREG(mode)) { + /* Nothing more to count */ + } else if (S_ISDIR(mode)) + stats.deleted_dirs++; +#ifdef SUPPORT_LINKS + else if (S_ISLNK(mode)) + stats.deleted_symlinks++; +#endif + else if (IS_DEVICE(mode)) + stats.deleted_symlinks++; + else + stats.deleted_specials++; + } ret = DR_SUCCESS; } else { if (S_ISDIR(mode) && errno == ENOTEMPTY) { @@ -214,10 +230,8 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) rsyserr(FERROR, errno, "delete_file: %s(%s) failed", what, fbuf); ret = DR_FAILURE; - } else { - deletion_count--; + } else ret = DR_SUCCESS; - } } check_ret: @@ -2285,7 +2299,7 @@ void generate_files(int f_out, const char *local_name) } } while ((cur_flist = cur_flist->next) != NULL); - if (read_batch) + if (read_batch && inc_recurse) write_ndx(f_out, NDX_DONE); if (delete_during) @@ -2306,6 +2320,8 @@ void generate_files(int f_out, const char *local_name) rprintf(FINFO, "generate_files phase=%d\n", phase); write_ndx(f_out, NDX_DONE); + write_del_stats(f_out); + /* Reduce round-trip lag-time for a useless delay-updates phase. */ if (protocol_version >= 29 && !delay_updates) write_ndx(f_out, NDX_DONE); @@ -2341,10 +2357,10 @@ void generate_files(int f_out, const char *local_name) && dir_tweaking && (!inc_recurse || delete_during == 2)) touch_up_dirs(dir_flist, -1); - if (max_delete >= 0 && deletion_count > max_delete) { + if (max_delete >= 0 && skipped_deletes) { rprintf(FWARNING, "Deletions stopped due to --max-delete limit (%d skipped)\n", - deletion_count - max_delete); + skipped_deletes); io_error |= IOERR_DEL_LIMIT; } diff --git a/main.c b/main.c index c050aaa1..2a44cf8a 100644 --- a/main.c +++ b/main.c @@ -171,6 +171,30 @@ static void wait_process_with_flush(pid_t pid, int *exit_code_ptr) *exit_code_ptr = WEXITSTATUS(status); } +void write_del_stats(int f) +{ + if (!INFO_GTE(STATS, 2) || protocol_version < 31) + return; + write_varint(f, stats.deleted_files - stats.deleted_dirs + - stats.deleted_symlinks - stats.deleted_devices + - stats.deleted_specials); + write_varint(f, stats.deleted_dirs); + write_varint(f, stats.deleted_symlinks); + write_varint(f, stats.deleted_devices); + write_varint(f, stats.deleted_specials); +} + +void read_del_stats(int f) +{ + if (!INFO_GTE(STATS, 2) || protocol_version < 31) + return; + stats.deleted_files = read_varint(f); + stats.deleted_files += stats.deleted_dirs = read_varint(f); + stats.deleted_files += stats.deleted_symlinks = read_varint(f); + stats.deleted_files += stats.deleted_devices = read_varint(f); + stats.deleted_files += stats.deleted_specials = read_varint(f); +} + /* This function gets called from all 3 processes. We want the client side * to actually output the text, but the sender is the only process that has * all the stats we need. So, if we're a client sender, we do the report. @@ -210,6 +234,7 @@ static void handle_stats(int f) write_varlong30(f, stats.flist_buildtime, 3); write_varlong30(f, stats.flist_xfertime, 3); } + write_del_stats(f); } return; } @@ -228,6 +253,8 @@ static void handle_stats(int f) stats.flist_buildtime = read_varlong30(f, 3); stats.flist_xfertime = read_varlong30(f, 3); } + if (!read_batch) + read_del_stats(f); } else if (write_batch) { /* The --read-batch process is going to be a client * receiver, so we need to give it the stats. */ @@ -238,16 +265,44 @@ static void handle_stats(int f) write_varlong30(batch_fd, stats.flist_buildtime, 3); write_varlong30(batch_fd, stats.flist_xfertime, 3); } + /* We don't write the del stats into the batch file -- they + * come from the generator when reading the batch. */ + } +} + +static void output_itemized_counts(const char *prefix, int *counts) +{ + static char *labels[] = { "reg", "dir", "link", "dev", "special" }; + char buf[1024], *pre = " ("; + int j, len = 0; + int total = counts[0]; + if (total) { + counts[0] -= counts[1] + counts[2] + counts[3] + counts[4]; + for (j = 0; j < 5; j++) { + if (counts[j]) { + len += snprintf(buf+len, sizeof buf - len - 2, + "%s%s: %s", + pre, labels[j], comma_num(counts[j])); + pre = ", "; + } + } + buf[len++] = ')'; } + buf[len] = '\0'; + rprintf(FINFO, "%s: %s%s\n", prefix, comma_num(total), buf); } static void output_summary(void) { if (INFO_GTE(STATS, 2)) { rprintf(FCLIENT, "\n"); - rprintf(FINFO,"Number of files: %s\n", comma_num(stats.num_files)); - rprintf(FINFO,"Number of files transferred: %s\n", - comma_num(stats.num_transferred_files)); + output_itemized_counts("Number of files", &stats.num_files); + if (protocol_version >= 29) + output_itemized_counts("Number of created files", &stats.created_files); + if (protocol_version >= 31) + output_itemized_counts("Number of deleted files", &stats.deleted_files); + rprintf(FINFO,"Number of regular files transferred: %s\n", + comma_num(stats.xferred_files)); rprintf(FINFO,"Total file size: %s bytes\n", human_num(stats.total_size)); rprintf(FINFO,"Total transferred file size: %s bytes\n", diff --git a/progress.c b/progress.c index e6cdd437..25be3749 100644 --- a/progress.c +++ b/progress.c @@ -73,7 +73,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, if (is_last) { int len = snprintf(eol, sizeof eol, " (xfr#%d, to-chk=%d/%d)\n", - stats.num_transferred_files, + stats.xferred_files, stats.num_files - current_file_index - 1, stats.num_files); if (INFO_GTE(PROGRESS, 2)) { diff --git a/receiver.c b/receiver.c index b342afa0..91642478 100644 --- a/receiver.c +++ b/receiver.c @@ -487,6 +487,21 @@ int recv_files(int f_in, char *local_name) if (preserve_xattrs && iflags & ITEM_REPORT_XATTR && !dry_run) set_file_attrs(fname, file, NULL, fname, 0); #endif + if (iflags & ITEM_IS_NEW) { + stats.created_files++; + if (S_ISREG(file->mode)) { + /* Nothing further to count. */ + } else if (S_ISDIR(file->mode)) + stats.created_dirs++; +#ifdef SUPPORT_LINKS + else if (S_ISLNK(file->mode)) + stats.created_symlinks++; +#endif + else if (IS_DEVICE(file->mode)) + stats.created_devices++; + else + stats.created_specials++; + } continue; } if (phase == 2) { @@ -516,11 +531,13 @@ int recv_files(int f_in, char *local_name) csum_length = SHORT_SUM_LENGTH; redoing = 0; } + if (iflags & ITEM_IS_NEW) + stats.created_files++; } if (!am_server && INFO_GTE(PROGRESS, 1)) set_current_file_index(file, ndx); - stats.num_transferred_files++; + stats.xferred_files++; stats.total_transferred_size += F_LENGTH(file); cleanup_got_literal = 0; @@ -808,6 +825,11 @@ int recv_files(int f_in, char *local_name) if (phase == 2 && delay_updates) /* for protocol_version < 29 */ handle_delayed_updates(local_name); + if (read_batch) { + read_int(batch_gen_fd); /* Discard -1 */ + read_del_stats(batch_gen_fd); + } + if (DEBUG_GTE(RECV, 1)) rprintf(FINFO,"recv_files finished\n"); diff --git a/rsync.h b/rsync.h index c288a1fe..a0f2d925 100644 --- a/rsync.h +++ b/rsync.h @@ -96,7 +96,7 @@ /* This is used when working on a new protocol version in CVS, and should * be a new non-zero value for each CVS change that affects the protocol. * It must ALWAYS be 0 when the protocol goes final (and NEVER before)! */ -#define SUBPROTOCOL_VERSION 1 +#define SUBPROTOCOL_VERSION 2 /* We refuse to interoperate with versions that are not in this range. * Note that we assume we'll work with later versions: the onus is on @@ -828,8 +828,10 @@ struct stats { int64 flist_buildtime; int64 flist_xfertime; int64 flist_size; - int num_files; - int num_transferred_files; + int num_files, num_dirs, num_symlinks, num_devices, num_specials; + int created_files, created_dirs, created_symlinks, created_devices, created_specials; + int deleted_files, deleted_dirs, deleted_symlinks, deleted_devices, deleted_specials; + int xferred_files; }; struct chmod_mode_struct; diff --git a/sender.c b/sender.c index 3937a10f..ed6a7368 100644 --- a/sender.c +++ b/sender.c @@ -201,6 +201,8 @@ void send_files(int f_in, int f_out) break; if (DEBUG_GTE(SEND, 1)) rprintf(FINFO, "send_files phase=%d\n", phase); + if (phase == 2) + read_del_stats(f_in); write_ndx(f_out, NDX_DONE); continue; } @@ -234,6 +236,21 @@ void send_files(int f_in, int f_out) maybe_log_item(file, iflags, itemizing, xname); write_ndx_and_attrs(f_out, ndx, iflags, fname, file, fnamecmp_type, xname, xlen); + if (iflags & ITEM_IS_NEW) { + stats.created_files++; + if (S_ISREG(file->mode)) { + /* Nothing further to count. */ + } else if (S_ISDIR(file->mode)) + stats.created_dirs++; +#ifdef SUPPORT_LINKS + else if (S_ISLNK(file->mode)) + stats.created_symlinks++; +#endif + else if (IS_DEVICE(file->mode)) + stats.created_devices++; + else + stats.created_specials++; + } continue; } if (phase == 2) { @@ -257,6 +274,8 @@ void send_files(int f_in, int f_out) append_mode = -append_mode; csum_length = SHORT_SUM_LENGTH; } + if (iflags & ITEM_IS_NEW) + stats.created_files++; } updating_basis_file = inplace && (protocol_version >= 29 @@ -264,7 +283,7 @@ void send_files(int f_in, int f_out) if (!am_server && INFO_GTE(PROGRESS, 1)) set_current_file_index(file, ndx); - stats.num_transferred_files++; + stats.xferred_files++; stats.total_transferred_size += F_LENGTH(file); if (!do_xfers) { /* log the transfer */ -- 2.34.1