Make idev, hlink and file_struct + strings use allocation

pools.

diff --git a/Makefile.in b/Makefile.in
index d60309f..03c7983 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -27,7 +27,7 @@ VERSION=@VERSION@
 
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o \
 
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h
 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o \

-       lib/permstring.o @LIBOBJS@
+       lib/permstring.o lib/pool_alloc.o @LIBOBJS@

 ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \
        zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \
        zlib/zutil.o zlib/adler32.o
 ZLIBOBJ=zlib/deflate.o zlib/infblock.o zlib/infcodes.o zlib/inffast.o \
        zlib/inflate.o zlib/inftrees.o zlib/infutil.o zlib/trees.o \
        zlib/zutil.o zlib/adler32.o

diff --git a/NEWS b/NEWS
index 61a30f9..82f734b 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -100,6 +100,10 @@ Changes since 2.6.0:
 
     * Less memory is used in the file list (a per-file savings).
 
 
     * Less memory is used in the file list (a per-file savings).
 

+    * Changed hardlink info and file_struct + strings to use
+      allocation pools.  This reduces memory use for large
+      filesets and permits freeing memory to the OS.  (J.W. Schultz) 
+

     * The 2 pipes used between the receiver and generator processes
       (which are forked on the same machine) were reduced to 1 pipe
       and the protocol improved so that (1) it is now impossible to
     * The 2 pipes used between the receiver and generator processes
       (which are forked on the same machine) were reduced to 1 pipe
       and the protocol improved so that (1) it is now impossible to

diff --git a/backup.c b/backup.c
index c1b8083..248ae90 100644 (file)
--- a/backup.c
+++ b/backup.c
@@ -189,6 +189,7 @@ static int keep_backup(char *fname)
                        backup_dir[--backup_dir_len] = '\0';
                if (verbose > 0)
                        rprintf(FINFO, "backup_dir is %s\n", backup_dir);
                        backup_dir[--backup_dir_len] = '\0';
                if (verbose > 0)
                        rprintf(FINFO, "backup_dir is %s\n", backup_dir);

+

                initialised = 1;
        }
 
                initialised = 1;
        }
 


@@ -199,7 +200,7 @@ static int keep_backup(char *fname)
        if (do_stat(fname, &st)) return 1;
 #endif
 
        if (do_stat(fname, &st)) return 1;
 #endif
 

-       file = make_file(fname, NO_EXCLUDES);
+       file = make_file(fname, NULL, NO_EXCLUDES);

 
        /* the file could have disappeared */
        if (!file) return 1;
 
        /* the file could have disappeared */
        if (!file) return 1;


@@ -282,7 +283,7 @@ static int keep_backup(char *fname)
                }
        }
        set_perms(keep_name, file, NULL, 0);
                }
        }
        set_perms(keep_name, file, NULL, 0);

-       free_file(file, FREE_STRUCT);
+       free(file);

 
        if (verbose > 1)
                rprintf(FINFO, "keep_backup %s -> %s\n", fname, keep_name);
 
        if (verbose > 1)
                rprintf(FINFO, "keep_backup %s -> %s\n", fname, keep_name);

diff --git a/batch.c b/batch.c
index d90c87b..dee68f0 100644 (file)
--- a/batch.c
+++ b/batch.c
@@ -136,9 +136,7 @@ struct file_list *create_flist_from_batch(void)
                exit_cleanup(1);
        }
 
                exit_cleanup(1);
        }
 

-       batch_flist = new(struct file_list);
-       if (!batch_flist)
-               out_of_memory("create_flist_from_batch");
+       batch_flist = flist_new(WITH_HLINK, "create_flist_from_batch");

 
        save_read = stats.total_read;
        save_pv = protocol_version;
 
        save_read = stats.total_read;
        save_pv = protocol_version;


@@ -150,9 +148,9 @@ struct file_list *create_flist_from_batch(void)
        for (i = 0; (flags = read_byte(f)) != 0; i++) {
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;
        for (i = 0; (flags = read_byte(f)) != 0; i++) {
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;

-               receive_file_entry(&batch_flist->files[i], flags, f);
+               receive_file_entry(&batch_flist->files[i], flags, batch_flist, f);

        }
        }

-       receive_file_entry(NULL, 0, 0); /* Signal that we're done. */
+       receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */

 
        protocol_version = save_pv;
        stats.total_read = save_read;
 
        protocol_version = save_pv;
        stats.total_read = save_read;

diff --git a/flist.c b/flist.c
index f21adb7..19931a5 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -71,18 +71,17 @@ extern struct exclude_struct **local_exclude_list;
 int io_error;
 
 static char empty_sum[MD4_SUM_LENGTH];
 int io_error;
 
 static char empty_sum[MD4_SUM_LENGTH];

-static unsigned int min_file_struct_len;
+static unsigned int file_struct_len;

 
 static void clean_flist(struct file_list *flist, int strip_root, int no_dups);
 static void output_flist(struct file_list *flist);
 
 
 static void clean_flist(struct file_list *flist, int strip_root, int no_dups);
 static void output_flist(struct file_list *flist);
 

-

 void init_flist(void)
 {
        struct file_struct f;
 
        /* Figure out how big the file_struct is without trailing padding */
 void init_flist(void)
 {
        struct file_struct f;
 
        /* Figure out how big the file_struct is without trailing padding */

-       min_file_struct_len = ((char*)&f.flags - (char*)&f) + sizeof f.flags;
+       file_struct_len = ((char*)&f.flags - (char*)&f) + sizeof f.flags;

 }
 
 
 }
 
 


@@ -507,7 +506,8 @@ void send_file_entry(struct file_struct *file, int f, unsigned short base_flags)
 
 
 
 
 
 

-void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
+void receive_file_entry(struct file_struct **fptr, unsigned short flags,
+    struct file_list *flist, int f)

 {
        static time_t modtime;
        static mode_t mode;
 {
        static time_t modtime;
        static mode_t mode;


@@ -520,7 +520,6 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
        char thisname[MAXPATHLEN];
        unsigned int l1 = 0, l2 = 0;
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
        char thisname[MAXPATHLEN];
        unsigned int l1 = 0, l2 = 0;
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;

-       int file_struct_len, idev_len;

        OFF_T file_length;
        char *basename, *dirname, *bp;
        struct file_struct *file;
        OFF_T file_length;
        char *basename, *dirname, *bp;
        struct file_struct *file;


@@ -614,24 +613,14 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
 #endif
                linkname_len = 0;
 
 #endif
                linkname_len = 0;
 

-#if SUPPORT_HARD_LINKS
-       if (preserve_hard_links && protocol_version < 28 && S_ISREG(mode))
-               flags |= XMIT_HAS_IDEV_DATA;
-       if (flags & XMIT_HAS_IDEV_DATA)
-               idev_len = sizeof (struct idev);
-       else
-#endif
-               idev_len = 0;
-

        sum_len = always_checksum && S_ISREG(mode) ? MD4_SUM_LENGTH : 0;
        sum_len = always_checksum && S_ISREG(mode) ? MD4_SUM_LENGTH : 0;

-       file_struct_len = idev_len? sizeof file[0] : min_file_struct_len;

 
        alloc_len = file_struct_len + dirname_len + basename_len
 
        alloc_len = file_struct_len + dirname_len + basename_len

-                 + linkname_len + sum_len + idev_len;
-       if (!(bp = new_array(char, alloc_len)))
-               out_of_memory("receive_file_entry");
+                 + linkname_len + sum_len;
+       bp = pool_alloc(flist->file_pool, alloc_len, "receive_file_entry");
+

        file = *fptr = (struct file_struct *)bp;
        file = *fptr = (struct file_struct *)bp;

-       memset(bp, 0, min_file_struct_len);
+       memset(bp, 0, file_struct_len);

        bp += file_struct_len;
 
        file->flags = flags & XMIT_TOP_DIR ? FLAG_TOP_DIR : 0;
        bp += file_struct_len;
 
        file->flags = flags & XMIT_TOP_DIR ? FLAG_TOP_DIR : 0;


@@ -641,13 +630,6 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
        file->uid = uid;
        file->gid = gid;
 
        file->uid = uid;
        file->gid = gid;
 

-#if SUPPORT_HARD_LINKS
-       if (idev_len) {
-               file->link_u.idev = (struct idev *)bp;
-               bp += idev_len;
-       }
-#endif
-

        if (dirname_len) {
                file->dirname = lastdir = bp;
                lastdir_len = dirname_len - 1;
        if (dirname_len) {
                file->dirname = lastdir = bp;
                lastdir_len = dirname_len - 1;


@@ -675,16 +657,24 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
 #endif
 
 #if SUPPORT_HARD_LINKS
 #endif
 
 #if SUPPORT_HARD_LINKS

-       if (idev_len) {
+       if (preserve_hard_links && protocol_version < 28 && S_ISREG(mode))
+               flags |= XMIT_HAS_IDEV_DATA;
+       if (flags & XMIT_HAS_IDEV_DATA && flist->hlink_pool) {
+               INO64_T inode;
+               file->link_u.idev = pool_talloc(flist->hlink_pool,
+                   struct idev, 1, "inode_table");

                if (protocol_version < 26) {
                        dev = read_int(f);
                if (protocol_version < 26) {
                        dev = read_int(f);

-                       file->F_INODE = read_int(f);
+                       inode = read_int(f);

                } else {
                        if (!(flags & XMIT_SAME_DEV))
                                dev = read_longint(f);
                } else {
                        if (!(flags & XMIT_SAME_DEV))
                                dev = read_longint(f);

-                       file->F_INODE = read_longint(f);
+                       inode = read_longint(f);
+               }
+               if (flist->hlink_pool) {
+                       file->F_INODE = inode;
+                       file->F_DEV = dev;

                }
                }

-               file->F_DEV = dev;

        }
 #endif
 
        }
 #endif
 


@@ -728,7 +718,8 @@ void receive_file_entry(struct file_struct **fptr, unsigned short flags, int f)
  * statting directories if we're not recursing, but this is not a very
  * important case.  Some systems may not have d_type.
  **/
  * statting directories if we're not recursing, but this is not a very
  * important case.  Some systems may not have d_type.
  **/

-struct file_struct *make_file(char *fname, int exclude_level)
+struct file_struct *make_file(char *fname,
+    struct file_list *flist, int exclude_level)

 {
        static char *lastdir;
        static int lastdir_len = -1;
 {
        static char *lastdir;
        static int lastdir_len = -1;


@@ -738,10 +729,10 @@ struct file_struct *make_file(char *fname, int exclude_level)
        char thisname[MAXPATHLEN];
        char linkname[MAXPATHLEN];
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;
        char thisname[MAXPATHLEN];
        char linkname[MAXPATHLEN];
        int alloc_len, basename_len, dirname_len, linkname_len, sum_len;

-       int file_struct_len, idev_len;

        char *basename, *dirname, *bp;
        unsigned short flags = 0;
 
        char *basename, *dirname, *bp;
        unsigned short flags = 0;
 

+

        if (strlcpy(thisname, fname, sizeof thisname)
            >= sizeof thisname - flist_dir_len) {
                rprintf(FINFO, "skipping overly long name: %s\n", fname);
        if (strlcpy(thisname, fname, sizeof thisname)
            >= sizeof thisname - flist_dir_len) {
                rprintf(FINFO, "skipping overly long name: %s\n", fname);


@@ -820,32 +811,20 @@ struct file_struct *make_file(char *fname, int exclude_level)
        linkname_len = 0;
 #endif
 
        linkname_len = 0;
 #endif
 

-#if SUPPORT_HARD_LINKS
-       if (preserve_hard_links) {
-               if (protocol_version < 28) {
-                       if (S_ISREG(st.st_mode))
-                               idev_len = sizeof (struct idev);
-                       else
-                               idev_len = 0;
-               } else {
-                       if (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
-                               idev_len = sizeof (struct idev);
-                       else
-                               idev_len = 0;
-               }
-       } else
-#endif
-               idev_len = 0;
-

        sum_len = always_checksum && S_ISREG(st.st_mode) ? MD4_SUM_LENGTH : 0;
        sum_len = always_checksum && S_ISREG(st.st_mode) ? MD4_SUM_LENGTH : 0;

-       file_struct_len = idev_len? sizeof file[0] : min_file_struct_len;

 
        alloc_len = file_struct_len + dirname_len + basename_len
 
        alloc_len = file_struct_len + dirname_len + basename_len

-                 + linkname_len + sum_len + idev_len;
-       if (!(bp = new_array(char, alloc_len)))
-               out_of_memory("receive_file_entry");
+           + linkname_len + sum_len;
+       if (flist) {
+               bp = pool_alloc(flist->file_pool, alloc_len,
+                   "receive_file_entry");
+       } else {
+               if (!(bp = new_array(char, alloc_len)))
+                       out_of_memory("receive_file_entry");
+       }
+

        file = (struct file_struct *)bp;
        file = (struct file_struct *)bp;

-       memset(bp, 0, min_file_struct_len);
+       memset(bp, 0, file_struct_len);

        bp += file_struct_len;
 
        file->flags = flags;
        bp += file_struct_len;
 
        file->flags = flags;


@@ -856,9 +835,20 @@ struct file_struct *make_file(char *fname, int exclude_level)
        file->gid = st.st_gid;
 
 #if SUPPORT_HARD_LINKS
        file->gid = st.st_gid;
 
 #if SUPPORT_HARD_LINKS

-       if (idev_len) {
-               file->link_u.idev = (struct idev *)bp;
-               bp += idev_len;
+       if (flist && flist->hlink_pool) {
+               if (protocol_version < 28) {
+                       if (S_ISREG(st.st_mode))
+                               file->link_u.idev = pool_talloc(
+                                   flist->hlink_pool, struct idev, 1,
+                                   "inode_table");
+               } else {
+                       if (!S_ISDIR(st.st_mode) && st.st_nlink > 1)
+                               file->link_u.idev = pool_talloc(
+                                   flist->hlink_pool, struct idev, 1,
+                                   "inode_table");
+               }
+       }
+       if (file->link_u.idev) {

                file->F_DEV = st.st_dev;
                file->F_INODE = st.st_ino;
        }
                file->F_DEV = st.st_dev;
                file->F_INODE = st.st_ino;
        }


@@ -913,9 +903,8 @@ void send_file_name(int f, struct file_list *flist, char *fname,
        extern int delete_excluded;
 
        /* f is set to -1 when calculating deletion file list */
        extern int delete_excluded;
 
        /* f is set to -1 when calculating deletion file list */

-       file = make_file(fname,
-                        f == -1 && delete_excluded? SERVER_EXCLUDES
-                                                  : ALL_EXCLUDES);
+       file = make_file(fname, flist,
+           f == -1 && delete_excluded? SERVER_EXCLUDES : ALL_EXCLUDES);

 
        if (!file)
                return;
 
        if (!file)
                return;


@@ -1034,7 +1023,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
        start_write = stats.total_written;
 
 
        start_write = stats.total_written;
 

-       flist = flist_new();
+       flist = flist_new(f == -1 ? WITHOUT_HLINK : WITH_HLINK,
+           "send_file_list");

 
        if (f != -1) {
                io_start_buffering_out(f);
 
        if (f != -1) {
                io_start_buffering_out(f);


@@ -1185,6 +1175,12 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                        finish_filelist_progress(flist);
        }
 
                        finish_filelist_progress(flist);
        }
 

+       if (flist->hlink_pool)
+       {
+               pool_destroy(flist->hlink_pool);
+               flist->hlink_pool = NULL;
+       }
+

        clean_flist(flist, 0, 0);
 
        if (f != -1) {
        clean_flist(flist, 0, 0);
 
        if (f != -1) {


@@ -1224,9 +1220,7 @@ struct file_list *recv_file_list(int f)
 
        start_read = stats.total_read;
 
 
        start_read = stats.total_read;
 

-       flist = new(struct file_list);
-       if (!flist)
-               goto oom;
+       flist = flist_new(WITH_HLINK, "recv_file_list");

 
        flist->count = 0;
        flist->malloced = 1000;
 
        flist->count = 0;
        flist->malloced = 1000;


@@ -1242,7 +1236,7 @@ struct file_list *recv_file_list(int f)
 
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;
 
                if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
                        flags |= read_byte(f) << 8;

-               receive_file_entry(&flist->files[i], flags, f);
+               receive_file_entry(&flist->files[i], flags, flist, f);

 
                if (S_ISREG(flist->files[i]->mode))
                        stats.total_size += flist->files[i]->length;
 
                if (S_ISREG(flist->files[i]->mode))
                        stats.total_size += flist->files[i]->length;


@@ -1256,7 +1250,7 @@ struct file_list *recv_file_list(int f)
                                f_name(flist->files[i]));
                }
        }
                                f_name(flist->files[i]));
                }
        }

-       receive_file_entry(NULL, 0, 0); /* Signal that we're done. */
+       receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */

 
        if (verbose > 2)
                rprintf(FINFO, "received %d names\n", flist->count);
 
        if (verbose > 2)
                rprintf(FINFO, "received %d names\n", flist->count);


@@ -1345,34 +1339,42 @@ int flist_find(struct file_list *flist, struct file_struct *f)
        return -1;
 }
 
        return -1;
 }
 

-

 /*
 /*

- * Free up any resources a file_struct has allocated, and optionally free
- * it up as well.
+ * Free up any resources a file_struct has allocated
+ * and clear the file.

  */
  */

-void free_file(struct file_struct *file, int free_the_struct)
+void clear_file(int i, struct file_list *flist)

 {
 {

-       if (free_the_struct)
-               free(file);
-       else
-               memset(file, 0, min_file_struct_len);
+       if (flist->hlink_pool && flist->files[i]->link_u.idev)
+               pool_free(flist->hlink_pool, 0, flist->files[i]->link_u.idev);
+       memset(flist->files[i], 0, file_struct_len);

 }
 
 
 /*
  * allocate a new file list
  */
 }
 
 
 /*
  * allocate a new file list
  */

-struct file_list *flist_new(void)
+struct file_list *flist_new(int with_hlink, char *msg)

 {
        struct file_list *flist;
 
        flist = new(struct file_list);
        if (!flist)
 {
        struct file_list *flist;
 
        flist = new(struct file_list);
        if (!flist)

-               out_of_memory("send_file_list");
+               out_of_memory(msg);

 
 

-       flist->count = 0;
-       flist->malloced = 0;
-       flist->files = NULL;
+       memset(flist, 0, sizeof (struct file_list));
+
+       if (!(flist->file_pool = pool_create(FILE_EXTENT, 0,
+           out_of_memory, POOL_INTERN)))
+               out_of_memory(msg);
+
+#if SUPPORT_HARD_LINKS
+       if (with_hlink && preserve_hard_links) {
+               if (!(flist->hlink_pool = pool_create(HLINK_EXTENT, 
+                   sizeof (struct idev), out_of_memory, POOL_INTERN)))
+                       out_of_memory(msg);
+       }
+#endif

 
        return flist;
 }
 
        return flist;
 }


@@ -1382,9 +1384,8 @@ struct file_list *flist_new(void)
  */
 void flist_free(struct file_list *flist)
 {
  */
 void flist_free(struct file_list *flist)
 {

-       int i;
-       for (i = 1; i < flist->count; i++)
-               free_file(flist->files[i], FREE_STRUCT);
+       pool_destroy(flist->file_pool);
+       pool_destroy(flist->hlink_pool);

        free(flist->files);
        free(flist);
 }
        free(flist->files);
        free(flist);
 }


@@ -1424,7 +1425,8 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
                         * else deletions will mysteriously fail with -R). */
                        if (flist->files[i]->flags & FLAG_TOP_DIR)
                                flist->files[prev_i]->flags |= FLAG_TOP_DIR;
                         * else deletions will mysteriously fail with -R). */
                        if (flist->files[i]->flags & FLAG_TOP_DIR)
                                flist->files[prev_i]->flags |= FLAG_TOP_DIR;

-                       free_file(flist->files[i], CLEAR_STRUCT);
+
+                       clear_file(i, flist);

                } else
                        prev_i = i;
        }
                } else
                        prev_i = i;
        }

diff --git a/hlink.c b/hlink.c
index 2dbc0d9..f42df53 100644 (file)
--- a/hlink.c
+++ b/hlink.c
@@ -46,26 +46,41 @@ int hlink_count;
 
 /* Analyze the data in the hlink_list[], remove items that aren't multiply
  * linked, and replace the dev+inode data with the hlindex+next linked list. */
 
 /* Analyze the data in the hlink_list[], remove items that aren't multiply
  * linked, and replace the dev+inode data with the hlindex+next linked list. */

-static void link_idev_data(void)
+static void link_idev_data(struct file_list *flist)

 {
        struct file_struct *head;
        int from, to, start;
 
 {
        struct file_struct *head;
        int from, to, start;
 

+       alloc_pool_t hlink_pool;
+       alloc_pool_t idev_pool = flist->hlink_pool;
+
+       hlink_pool = pool_create(128 * 1024, sizeof (struct hlink),
+           out_of_memory, POOL_INTERN);
+

        for (from = to = 0; from < hlink_count; from++) {
                start = from;
                head = hlink_list[start];
                while (from < hlink_count-1
                    && LINKED(hlink_list[from], hlink_list[from+1])) {
        for (from = to = 0; from < hlink_count; from++) {
                start = from;
                head = hlink_list[start];
                while (from < hlink_count-1
                    && LINKED(hlink_list[from], hlink_list[from+1])) {

+                       pool_free(idev_pool, 0, hlink_list[from]->link_u.idev);
+                       hlink_list[from]->link_u.links = pool_talloc(hlink_pool,
+                           struct hlink, 1, "hlink_list");
+

                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = hlink_list[from+1];
                        from++;
                }
                if (from > start) {
                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = hlink_list[from+1];
                        from++;
                }
                if (from > start) {

+                       pool_free(idev_pool, 0, hlink_list[from]->link_u.idev);
+                       hlink_list[from]->link_u.links = pool_talloc(hlink_pool,
+                           struct hlink, 1, "hlink_list");
+

                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = head;
                        hlink_list[from]->flags |= FLAG_HLINK_EOL;
                        hlink_list[to++] = head;
                } else {
                        hlink_list[from]->F_HLINDEX = to;
                        hlink_list[from]->F_NEXT = head;
                        hlink_list[from]->flags |= FLAG_HLINK_EOL;
                        hlink_list[to++] = head;
                } else {

+                       pool_free(idev_pool, 0, head->link_u.idev);

                        head->link_u.idev = NULL;
                }
        }
                        head->link_u.idev = NULL;
                }
        }


@@ -73,12 +88,16 @@ static void link_idev_data(void)
        if (!to) {
                free(hlink_list);
                hlink_list = NULL;
        if (!to) {
                free(hlink_list);
                hlink_list = NULL;

+               pool_destroy(hlink_pool);
+               hlink_pool = NULL;

        } else {
                hlink_count = to;
                if (!(hlink_list = realloc_array(hlink_list,
                    struct file_struct *, hlink_count)))
                        out_of_memory("init_hard_links");
        }
        } else {
                hlink_count = to;
                if (!(hlink_list = realloc_array(hlink_list,
                    struct file_struct *, hlink_count)))
                        out_of_memory("init_hard_links");
        }

+       flist->hlink_pool = hlink_pool;
+       pool_destroy(idev_pool);

 }
 #endif
 
 }
 #endif
 


@@ -109,7 +128,7 @@ void init_hard_links(struct file_list *flist)
                free(hlink_list);
                hlink_list = NULL;
        } else
                free(hlink_list);
                hlink_list = NULL;
        } else

-               link_idev_data();
+               link_idev_data(flist);

 #endif
 }
 
 #endif
 }
 

diff --git a/receiver.c b/receiver.c
index da89819..ca6bf23 100644 (file)
--- a/receiver.c
+++ b/receiver.c
@@ -305,6 +305,12 @@ int recv_files(int f_in,struct file_list *flist,char *local_name)
                rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
        }
 
                rprintf(FINFO,"recv_files(%d) starting\n",flist->count);
        }
 

+       if (flist->hlink_pool)
+       {
+               pool_destroy(flist->hlink_pool);
+               flist->hlink_pool = NULL;
+       }
+

        while (1) {
                cleanup_disable();
 
        while (1) {
                cleanup_disable();
 

diff --git a/rsync.h b/rsync.h
index aa22cfe..37a5d53 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -112,8 +112,6 @@
 #define FULL_FLUSH     1
 #define NORMAL_FLUSH   0
 
 #define FULL_FLUSH     1
 #define NORMAL_FLUSH   0
 

-#define CLEAR_STRUCT   0
-#define FREE_STRUCT    1

 
 /* Log-message categories.  FLOG is only used on the daemon side to
  * output messages to the log file. */
 
 /* Log-message categories.  FLOG is only used on the daemon side to
  * output messages to the log file. */


@@ -254,6 +252,7 @@ enum msgcode {
 
 #include <assert.h>
 
 
 #include <assert.h>
 

+#include "lib/pool_alloc.h"

 
 #define BOOL int
 
 
 #define BOOL int
 


@@ -434,10 +433,27 @@ struct file_struct {
  */
 #define FLIST_START    (32 * 1024)
 #define FLIST_LINEAR   (FLIST_START * 512)
  */
 #define FLIST_START    (32 * 1024)
 #define FLIST_LINEAR   (FLIST_START * 512)

+/*
+ * Extent size for allocation pools A minimum size of 128KB
+ * is needed to mmap them so that freeing will release the
+ * space to the OS.
+ *
+ * Larger sizes reduce leftover fragments and speed free calls
+ * (when they happen) Smaller sizes increase the chance of
+ * freed allocations freeing whole extents.
+ */
+
+#define FILE_EXTENT    (256 * 1024)
+#define HLINK_EXTENT   (128 * 1024)
+
+#define WITH_HLINK     1
+#define WITHOUT_HLINK  0

 
 struct file_list {
        int count;
        int malloced;
 
 struct file_list {
        int count;
        int malloced;

+       alloc_pool_t file_pool;
+       alloc_pool_t hlink_pool;

        struct file_struct **files;
 };
 
        struct file_struct **files;
 };