Unified the file-list pool used in incremental recursion mode so that

we use less memory, especially in small transfers.  As file lists are
discarded, we use the new pool_boundary() and pool_free_old() functions
to discard pool extents that are no longer needed.

diff --git a/flist.c b/flist.c
index 04c3c02..e6a8085 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -308,6 +308,15 @@ static void flist_expand(struct file_list *flist, int extra)
                out_of_memory("flist_expand");
 }
 
+static void flist_done_allocating(struct file_list *flist)
+{
+       void *ptr = pool_boundary(flist->file_pool, 8*1024);
+       if (flist->pool_boundary == ptr)
+               flist->pool_boundary = NULL; /* list didn't use any pool memory */
+       else
+               flist->pool_boundary = ptr;
+}
+
 int push_pathname(const char *dir, int len)
 {
        if (dir == pathname)
@@ -1562,6 +1571,7 @@ void send_extra_file_list(int f, int at_least)
                clean_flist(flist, 0);
 
                add_dirs_to_tree(send_dir_ndx, flist, dir_count - dstart);
+               flist_done_allocating(flist);
 
                file_total += flist->count;
                future_cnt += flist->count;
@@ -1895,6 +1905,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
        if (inc_recurse) {
                add_dirs_to_tree(-1, flist, dir_count);
+               flist_done_allocating(flist);
                if (send_dir_ndx < 0) {
                        write_ndx(f, NDX_FLIST_EOF);
                        flist_eof = 1;
@@ -2001,7 +2012,9 @@ struct file_list *recv_file_list(int f)
                }
        }
 
-       if (!inc_recurse && f >= 0)
+       if (inc_recurse)
+               flist_done_allocating(flist);
+       else if (f >= 0)
                recv_id_list(f, flist);
 
        clean_flist(flist, relative_paths);
@@ -2137,34 +2150,42 @@ struct file_list *flist_new(int flags, char *msg)
 
        memset(flist, 0, sizeof flist[0]);
 
-       if (!(flags & FLIST_TEMP)) {
-               if (first_flist) {
-                       flist->ndx_start = first_flist->prev->ndx_start
-                                        + first_flist->prev->count;
-               }
+       if (flags & FLIST_TEMP) {
+               if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0,
+                                               out_of_memory, POOL_INTERN)))
+                       out_of_memory(msg);
+       } else {
                /* This is a doubly linked list with prev looping back to
                 * the end of the list, but the last next pointer is NULL. */
-               if (!first_flist)
+               if (!first_flist) {
+                       flist->file_pool = pool_create(NORMAL_EXTENT, 0,
+                                               out_of_memory, POOL_INTERN);
+                       if (!flist->file_pool)
+                               out_of_memory(msg);
+
                        first_flist = cur_flist = flist->prev = flist;
-               else {
+               } else {
+                       flist->file_pool = first_flist->file_pool;
+
+                       flist->ndx_start = first_flist->prev->ndx_start
+                                        + first_flist->prev->count;
+
                        flist->prev = first_flist->prev;
                        flist->prev->next = first_flist->prev = flist;
                }
+               flist->pool_boundary = pool_boundary(flist->file_pool, 0);
                flist_cnt++;
        }
 
-       if (!(flist->file_pool = pool_create(FILE_EXTENT, 0, out_of_memory, POOL_INTERN)))
-               out_of_memory(msg);
-
        return flist;
 }
 
 /* Free up all elements in a flist. */
 void flist_free(struct file_list *flist)
 {
-       if (!flist->prev)
-               ; /* Was FLIST_TEMP dir-list. */
-       else if (flist == flist->prev) {
+       if (!flist->prev) {
+               /* Was FLIST_TEMP dir-list. */
+       } else if (flist == flist->prev) {
                first_flist = cur_flist = NULL;
                file_total = 0;
                flist_cnt = 0;
@@ -2183,7 +2204,11 @@ void flist_free(struct file_list *flist)
                flist_cnt--;
        }
 
-       pool_destroy(flist->file_pool);
+       if (!flist->prev || !flist_cnt)
+               pool_destroy(flist->file_pool);
+       else
+               pool_free_old(flist->file_pool, flist->pool_boundary);
+
        if (flist->sorted && flist->sorted != flist->files)
                free(flist->sorted);
        free(flist->files);