Changed the memory implementation of --delay-updates so that it

only allocates memory to store needed bits, it sparsely allocates
the bits in 16KB chunks, and the bit-finding loop works in a more
efficient manner.

diff --git a/receiver.c b/receiver.c
index 719a58b..1779d17 100644 (file)
--- a/receiver.c
+++ b/receiver.c
@@ -79,6 +79,69 @@ void delete_files(struct file_list *flist)
        }
 }
 
+#define SLOT_SIZE      (16*1024)       /* Desired size in bytes */
+#define PER_SLOT_BITS  (SLOT_SIZE * 8) /* Number of bits per slot */
+#define PER_SLOT_INTS  (SLOT_SIZE / 4) /* Number of int32s per slot */
+
+static uint32 **delayed_bits = NULL;
+static int delayed_slot_cnt = 0;
+
+static void init_delayed_bits(int max_ndx)
+{
+       delayed_slot_cnt = (max_ndx + PER_SLOT_BITS - 1) / PER_SLOT_BITS;
+
+       if (!(delayed_bits = (uint32**)calloc(delayed_slot_cnt, sizeof (uint32*))))
+               out_of_memory("set_delayed_bit");
+}
+
+static void set_delayed_bit(int ndx)
+{
+       int slot = ndx / PER_SLOT_BITS;
+       ndx %= PER_SLOT_BITS;
+
+       if (!delayed_bits[slot]) {
+               if (!(delayed_bits[slot] = (uint32*)calloc(PER_SLOT_INTS, 4)))
+                       out_of_memory("set_delayed_bit");
+       }
+
+       delayed_bits[slot][ndx/32] |= 1u << (ndx % 32);
+}
+
+/* Call this with -1 to start checking from 0.  Returns -1 at the end. */
+static int next_delayed_bit(int after)
+{
+       uint32 bits, mask;
+       int i, ndx = after + 1;
+       int slot = ndx / PER_SLOT_BITS;
+       ndx %= PER_SLOT_BITS;
+
+       mask = (1u << (ndx % 32)) - 1;
+       for (i = ndx / 32; slot < delayed_slot_cnt; slot++, i = mask = 0) {
+               if (!delayed_bits[slot])
+                       continue;
+               for ( ; i < PER_SLOT_INTS; i++, mask = 0) {
+                       if (!(bits = delayed_bits[slot][i] & ~mask))
+                               continue;
+                       /* The xor magic figures out the lowest enabled bit in
+                        * bits, and the switch quickly computes log2(bit). */
+                       switch (bits ^ (bits & (bits-1))) {
+#define LOG2(n) case 1u << n: return slot*PER_SLOT_BITS + i*32 + n
+                           LOG2(0);  LOG2(1);  LOG2(2);  LOG2(3);
+                           LOG2(4);  LOG2(5);  LOG2(6);  LOG2(7);
+                           LOG2(8);  LOG2(9);  LOG2(10); LOG2(11);
+                           LOG2(12); LOG2(13); LOG2(14); LOG2(15);
+                           LOG2(16); LOG2(17); LOG2(18); LOG2(19);
+                           LOG2(20); LOG2(21); LOG2(22); LOG2(23);
+                           LOG2(24); LOG2(25); LOG2(26); LOG2(27);
+                           LOG2(28); LOG2(29); LOG2(30); LOG2(31);
+                       }
+                       return -1; /* impossible... */
+               }
+       }
+
+       return -1;
+}
+
 
 /*
  * get_tmpname() - create a tmp filename for a given filename
@@ -315,7 +378,6 @@ int recv_files(int f_in, struct file_list *flist, char *local_name,
        char fnametmp[MAXPATHLEN];
        char *fnamecmp, *partialptr, numbuf[4];
        char fnamecmpbuf[MAXPATHLEN];
-       uchar *delayed_bits = NULL;
        struct file_struct *file;
        struct stats initial_stats;
        int save_make_backups = make_backups;
@@ -331,12 +393,8 @@ int recv_files(int f_in, struct file_list *flist, char *local_name,
                flist->hlink_pool = NULL;
        }
 
-       if (delay_updates) {
-               int sz = (flist->count + 7) / 8;
-               if (!(delayed_bits = new_array(uchar, sz)))
-                       out_of_memory("recv_files");
-               memset(delayed_bits, 0, sz);
-       }
+       if (delay_updates)
+               init_delayed_bits(flist->count);
 
        while (1) {
                cleanup_disable();
@@ -587,8 +645,10 @@ int recv_files(int f_in, struct file_list *flist, char *local_name,
                    && handle_partial_dir(partialptr, PDIR_CREATE)) {
                        finish_transfer(partialptr, fnametmp, file, recv_ok,
                                        !partial_dir);
-                       if (delay_updates && recv_ok)
-                               delayed_bits[i/8] |= 1 << (i % 8);
+                       if (delay_updates && recv_ok) {
+                               set_delayed_bit(i);
+                               recv_ok = -1;
+                       }
                } else {
                        partialptr = NULL;
                        do_unlink(fnametmp);
@@ -597,9 +657,7 @@ int recv_files(int f_in, struct file_list *flist, char *local_name,
                cleanup_disable();
 
                if (recv_ok) {
-                       if (delay_updates && delayed_bits[i/8] & (1 << (i % 8)))
-                               ;
-                       else if (remove_sent_files) {
+                       if (remove_sent_files && recv_ok > 0) {
                                SIVAL(numbuf, 0, i);
                                send_msg(MSG_SUCCESS, numbuf, 4);
                        }
@@ -635,11 +693,8 @@ int recv_files(int f_in, struct file_list *flist, char *local_name,
        make_backups = save_make_backups;
 
        if (delay_updates) {
-               for (i = 0; i < flist->count; i++) {
+               for (i = -1; (i = next_delayed_bit(i)) >= 0; ) {
                        struct file_struct *file = flist->files[i];
-                       if (!file->basename
-                        || !(delayed_bits[i/8] & (1 << (i % 8))))
-                               continue;
                        fname = local_name ? local_name : f_name(file);
                        partialptr = partial_dir_fname(fname);
                        if (partialptr) {