Updated indexing for accurate progress and improved raw ndx values.

The sending side now has a sorted file-list in iconv mode so that it
can output progress in sorted order.  Simplified the over-the-wire
index values to ensure both sides will always agree on the values.
Optimized the allocation of the dir_flist->sorted array on the
receiving side with --iconv and incremental recursion.

diff --git a/flist.c b/flist.c
index b8cc9fe..0893b4a 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -121,9 +121,8 @@ static char tmp_sum[MAX_DIGEST_LEN];
 static char empty_sum[MAX_DIGEST_LEN];
 static int flist_count_offset; /* for --delete --progress */
 static int dir_count = 0;
-static int high_hlink_ndx;
 
-static void clean_flist(struct file_list *flist, int strip_root);
+static void flist_sort_and_clean(struct file_list *flist, int strip_root);
 static void output_flist(struct file_list *flist);
 
 void init_flist(void)
@@ -457,8 +456,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_
                        struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino);
                        first_hlink_ndx = (int32)(long)np->data - 1;
                        if (first_hlink_ndx < 0) {
-                               high_hlink_ndx = ndx + first_ndx;
-                               np->data = (void*)(long)(high_hlink_ndx + 1);
+                               np->data = (void*)(long)(first_ndx + ndx + 1);
                                xflags |= XMIT_HLINK_FIRST;
                        }
                        xflags |= XMIT_HLINKED;
@@ -939,8 +937,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
        if (preserve_hard_links && xflags & XMIT_HLINKED) {
                if (protocol_version >= 30) {
                        if (xflags & XMIT_HLINK_FIRST) {
-                               high_hlink_ndx = flist->ndx_start + flist->used;
-                               F_HL_GNUM(file) = high_hlink_ndx;
+                               F_HL_GNUM(file) = flist->ndx_start + flist->used;
                        } else
                                F_HL_GNUM(file) = first_hlink_ndx;
                } else {
@@ -1719,11 +1716,7 @@ void send_extra_file_list(int f, int at_least)
                } else
                        flist->sorted = flist->files;
 
-               clean_flist(flist, 0);
-
-               flist->ndx_end = flist->ndx_start + flist->used - 1;
-               if (!need_unsorted_flist)
-                       flist->ndx_end -= (dir_count - dstart);
+               flist_sort_and_clean(flist, 0);
 
                add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart);
                flist_done_allocating(flist);
@@ -2021,27 +2014,15 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
         * recursion mode, the sender marks duplicate dirs so that it can
         * send them together in a single file-list. */
        if (need_unsorted_flist) {
-               if (inc_recurse) {
-                       if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
-                               out_of_memory("send_file_list");
-                       memcpy(flist->sorted, flist->files,
-                              flist->used * sizeof (struct file_struct*));
-                       clean_flist(flist, 0);
-               } else {
-                       flist->sorted = flist->files;
-                       flist->low = 0;
-                       flist->high = flist->used - 1;
-               }
-       } else {
+               if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
+                       out_of_memory("send_file_list");
+               memcpy(flist->sorted, flist->files,
+                      flist->used * sizeof (struct file_struct*));
+       } else
                flist->sorted = flist->files;
-               clean_flist(flist, 0);
-       }
+       flist_sort_and_clean(flist, 0);
        file_total += flist->used;
 
-       /* We don't subtract dir_count for the first send since we
-        * might have one or more dot dirs which need to get sent. */
-       flist->ndx_end = flist->ndx_start + flist->used - 1;
-
        if (!numeric_ids && !inc_recurse)
                send_id_list(f);
 
@@ -2140,10 +2121,6 @@ struct file_list *recv_file_list(int f)
        }
        file_total += flist->used;
 
-       flist->ndx_end = flist->ndx_start + flist->used - 1;
-       if (inc_recurse && !need_unsorted_flist && flist->ndx_start > 1)
-               flist->ndx_end -= dir_flist->used - dstart;
-
        if (verbose > 2)
                rprintf(FINFO, "received %d names\n", flist->used);
 
@@ -2161,9 +2138,13 @@ struct file_list *recv_file_list(int f)
                memcpy(flist->sorted, flist->files,
                       flist->used * sizeof (struct file_struct*));
                if (inc_recurse && dir_flist->used > dstart) {
-                       dir_flist->sorted = realloc_array(dir_flist->sorted,
-                                               struct file_struct *,
-                                               dir_flist->used);
+                       static int dir_flist_malloced = 0;
+                       if (dir_flist_malloced < dir_flist->malloced) {
+                               dir_flist->sorted = realloc_array(dir_flist->sorted,
+                                                       struct file_struct *,
+                                                       dir_flist->malloced);
+                               dir_flist_malloced = dir_flist->malloced;
+                       }
                        memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
                               (dir_flist->used - dstart) * sizeof (struct file_struct*));
                        fsort(dir_flist->sorted + dstart, dir_flist->used - dstart);
@@ -2181,7 +2162,7 @@ struct file_list *recv_file_list(int f)
        else if (f >= 0)
                recv_id_list(f, flist);
 
-       clean_flist(flist, relative_paths);
+       flist_sort_and_clean(flist, relative_paths);
 
        if (protocol_version < 30) {
                /* Recv the io_error flag */
@@ -2321,18 +2302,19 @@ struct file_list *flist_new(int flags, char *msg)
                        if (!flist->file_pool)
                                out_of_memory(msg);
 
-                       flist->ndx_start = inc_recurse ? 1 : 0;
+                       flist->ndx_start = flist->flist_num = inc_recurse ? 1 : 0;
 
                        first_flist = cur_flist = flist->prev = flist;
                } else {
+                       struct file_list *prev = first_flist->prev;
+
                        flist->file_pool = first_flist->file_pool;
 
-                       flist->ndx_start = first_flist->prev->ndx_end + 2;
-                       if (flist->ndx_start <= high_hlink_ndx)
-                               flist->ndx_start = high_hlink_ndx + 1;
+                       flist->ndx_start = prev->ndx_start + prev->used + 1;
+                       flist->flist_num = prev->flist_num + 1;
 
-                       flist->prev = first_flist->prev;
-                       flist->prev->next = first_flist->prev = flist;
+                       flist->prev = prev;
+                       prev->next = first_flist->prev = flist;
                }
                flist->pool_boundary = pool_boundary(flist->file_pool, 0);
                flist_cnt++;
@@ -2378,7 +2360,7 @@ void flist_free(struct file_list *flist)
 
 /* This routine ensures we don't have any duplicate names in our file list.
  * duplicate names can cause corruption because of the pipelining. */
-static void clean_flist(struct file_list *flist, int strip_root)
+static void flist_sort_and_clean(struct file_list *flist, int strip_root)
 {
        char fbuf[MAXPATHLEN];
        int i, prev_i;
@@ -2558,8 +2540,8 @@ static void output_flist(struct file_list *flist)
        const char *who = who_am_i();
        int i;
 
-       rprintf(FINFO, "[%s] flist start=%d, end=%d, used=%d, low=%d, high=%d\n",
-               who, flist->ndx_start, flist->ndx_end, flist->used, flist->low, flist->high);
+       rprintf(FINFO, "[%s] flist start=%d, used=%d, low=%d, high=%d\n",
+               who, flist->ndx_start, flist->used, flist->low, flist->high);
        for (i = 0; i < flist->used; i++) {
                file = flist->files[i];
                if ((am_root || am_sender) && uid_ndx) {
@@ -2809,7 +2791,7 @@ struct file_list *get_dirlist(char *dirname, int dlen, int ignore_filter_rules)
 
        prune_empty_dirs = 0;
        dirlist->sorted = dirlist->files;
-       clean_flist(dirlist, 0);
+       flist_sort_and_clean(dirlist, 0);
        prune_empty_dirs = save_prune_empty_dirs;
 
        if (verbose > 3)

diff --git a/progress.c b/progress.c
index 476dd4d..c8ab1a1 100644 (file)
--- a/progress.c
+++ b/progress.c
@@ -22,8 +22,10 @@
 
 #include "rsync.h"
 
-extern struct stats stats;
 extern int am_server;
+extern int need_unsorted_flist;
+extern struct stats stats;
+extern struct file_list *cur_flist;
 
 #define PROGRESS_HISTORY_SECS 5
 
@@ -41,6 +43,7 @@ struct progress_history {
 static struct progress_history ph_start;
 static struct progress_history ph_list[PROGRESS_HISTORY_SECS];
 static int newest_hpos, oldest_hpos;
+static int current_file_index;
 
 static unsigned long msdiff(struct timeval *t1, struct timeval *t2)
 {
@@ -104,7 +107,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now,
        if (is_last) {
                snprintf(eol, sizeof eol, " (xfer#%d, to-check=%d/%d)\n",
                        stats.num_transferred_files,
-                       stats.num_files - stats.current_file_index - 1,
+                       stats.num_files - current_file_index - 1,
                        stats.num_files);
        } else
                strlcpy(eol, "\r", sizeof eol);
@@ -112,6 +115,15 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now,
                human_num(ofs), pct, rate, units, rembuf, eol);
 }
 
+void set_current_file_index(struct file_struct *file, int ndx)
+{
+       if (need_unsorted_flist)
+               current_file_index = flist_find(cur_flist, file) + cur_flist->ndx_start;
+       else
+               current_file_index = ndx;
+       current_file_index -= cur_flist->flist_num;
+}
+
 void end_progress(OFF_T size)
 {
        if (!am_server) {

diff --git a/receiver.c b/receiver.c
index 387567b..3834ce2 100644 (file)
--- a/receiver.c
+++ b/receiver.c
@@ -481,9 +481,11 @@ int recv_files(int f_in, char *local_name)
                        }
                }
 
-               stats.current_file_index = ndx;
+               if (!am_server && do_progress)
+                       set_current_file_index(file, ndx);
                stats.num_transferred_files++;
                stats.total_transferred_size += F_LENGTH(file);
+
                cleanup_got_literal = 0;
 
                if (server_filter_list.head

diff --git a/rsync.c b/rsync.c
index 47b0ca1..1654c30 100644 (file)
--- a/rsync.c
+++ b/rsync.c
@@ -264,12 +264,13 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
        }
 
        if (!(flist = flist_for_ndx(ndx))) {
+               int start, used;
          invalid_ndx:
+               start = first_flist ? first_flist->ndx_start : 0;
+               used = first_flist ? first_flist->used : 0;
                rprintf(FERROR,
                        "Invalid file index: %d (%d - %d) with iflags %x [%s]\n",
-                       ndx, first_flist ? first_flist->ndx_start - 1 : -1,
-                       first_flist ? first_flist->prev->ndx_end : -1,
-                       iflags, who_am_i());
+                       ndx, start - 1, start + used -1, iflags, who_am_i());
                exit_cleanup(RERR_PROTOCOL);
        }
        cur_flist = flist;
@@ -572,7 +573,7 @@ struct file_list *flist_for_ndx(int ndx)
                        return NULL;
                flist = flist->prev;
        }
-       while (ndx > flist->ndx_end) {
+       while (ndx >= flist->ndx_start + flist->used) {
                if (!(flist = flist->next))
                        return NULL;
        }

diff --git a/rsync.h b/rsync.h
index 8c9673d..2592230 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -93,7 +93,7 @@
 /* This is used when working on a new protocol version in CVS, and should
  * be a new non-zero value for each CVS change that affects the protocol.
  * It must ALWAYS be 0 when the protocol goes final! */
-#define SUBPROTOCOL_VERSION 15
+#define SUBPROTOCOL_VERSION 16
 
 /* We refuse to interoperate with versions that are not in this range.
  * Note that we assume we'll work with later versions: the onus is on
@@ -723,7 +723,7 @@ struct file_list {
        int used, malloced;
        int low, high;  /* 0-relative index values excluding empties */
        int ndx_start;  /* the start offset for inc_recurse mode */
-       int ndx_end;    /* the end offset for inc_recurse mode */
+       int flist_num;  /* 1-relative file_list number or 0 */
        int parent_ndx; /* dir_flist index of parent directory */
        int in_progress, to_redo;
 };
@@ -814,7 +814,6 @@ struct stats {
        int64 flist_size;
        int num_files;
        int num_transferred_files;
-       int current_file_index;
 };
 
 struct chmod_mode_struct;

diff --git a/sender.c b/sender.c
index 33d5b94..fc9c47c 100644 (file)
--- a/sender.c
+++ b/sender.c
@@ -265,7 +265,8 @@ void send_files(int f_in, int f_out)
                updating_basis_file = inplace && (protocol_version >= 29
                        ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0);
 
-               stats.current_file_index = ndx;
+               if (!am_server && do_progress)
+                       set_current_file_index(file, ndx);
                stats.num_transferred_files++;
                stats.total_transferred_size += F_LENGTH(file);