Adding the --usermap/--groupmap/--chown options.
[rsync/rsync.git] / generator.c
index a09480a..1db0fe0 100644 (file)
@@ -21,8 +21,8 @@
  */
 
 #include "rsync.h"
+#include "inums.h"
 
-extern int verbose;
 extern int dry_run;
 extern int do_xfers;
 extern int stdout_format_has_i;
@@ -31,7 +31,6 @@ extern int am_root;
 extern int am_server;
 extern int am_daemon;
 extern int inc_recurse;
-extern int do_progress;
 extern int relative_paths;
 extern int implied_dirs;
 extern int keep_dirlinks;
@@ -55,6 +54,7 @@ extern int ignore_errors;
 extern int remove_source_files;
 extern int delay_updates;
 extern int update_only;
+extern int human_readable;
 extern int ignore_existing;
 extern int ignore_non_existing;
 extern int inplace;
@@ -76,7 +76,6 @@ extern int fuzzy_basis;
 extern int always_checksum;
 extern int checksum_len;
 extern char *partial_dir;
-extern char *basis_dir[];
 extern int compare_dest;
 extern int copy_dest;
 extern int link_dest;
@@ -96,19 +95,20 @@ extern uid_t our_uid;
 extern char *backup_dir;
 extern char *backup_suffix;
 extern int backup_suffix_len;
+extern char *basis_dir[MAX_BASIS_DIRS+1];
 extern struct file_list *cur_flist, *first_flist, *dir_flist;
-extern struct filter_list_struct server_filter_list;
+extern struct filter_list_struct daemon_filter_list;
 
 int ignore_perishable = 0;
 int non_perishable_cnt = 0;
 int maybe_ATTRS_REPORT = 0;
 
 static dev_t dev_zero;
-static int deletion_count = 0; /* used to implement --max-delete */
+static int skipped_deletes = 0;
 static int deldelay_size = 0, deldelay_cnt = 0;
 static char *deldelay_buf = NULL;
 static int deldelay_fd = -1;
-static int lull_mod;
+static int loopchk_limit;
 static int dir_tweaking;
 static int symlink_timeset_failed_flags;
 static int need_retouch_dir_times;
@@ -160,7 +160,7 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags)
        char *what;
        int ok;
 
-       if (verbose > 2) {
+       if (DEBUG_GTE(DEL, 2)) {
                rprintf(FINFO, "delete_item(%s) mode=%o flags=%d\n",
                        fbuf, (int)mode, (int)flags);
        }
@@ -187,8 +187,10 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags)
                /* OK: try to delete the directory. */
        }
 
-       if (!(flags & DEL_MAKE_ROOM) && max_delete >= 0 && ++deletion_count > max_delete)
+       if (!(flags & DEL_MAKE_ROOM) && max_delete >= 0 && stats.deleted_files >= max_delete) {
+               skipped_deletes++;
                return DR_AT_LIMIT;
+       }
 
        if (S_ISDIR(mode)) {
                what = "rmdir";
@@ -202,8 +204,22 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags)
        }
 
        if (ok) {
-               if (!(flags & DEL_MAKE_ROOM))
+               if (!(flags & DEL_MAKE_ROOM)) {
                        log_delete(fbuf, mode);
+                       stats.deleted_files++;
+                       if (S_ISREG(mode)) {
+                               /* Nothing more to count */
+                       } else if (S_ISDIR(mode))
+                               stats.deleted_dirs++;
+#ifdef SUPPORT_LINKS
+                       else if (S_ISLNK(mode))
+                               stats.deleted_symlinks++;
+#endif
+                       else if (IS_DEVICE(mode))
+                               stats.deleted_symlinks++;
+                       else
+                               stats.deleted_specials++;
+               }
                ret = DR_SUCCESS;
        } else {
                if (S_ISDIR(mode) && errno == ENOTEMPTY) {
@@ -214,10 +230,8 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags)
                        rsyserr(FERROR, errno, "delete_file: %s(%s) failed",
                                what, fbuf);
                        ret = DR_FAILURE;
-               } else {
-                       deletion_count--;
+               } else
                        ret = DR_SUCCESS;
-               }
        }
 
   check_ret:
@@ -251,7 +265,7 @@ static enum delret delete_dir_contents(char *fname, uint16 flags)
        int j, dlen;
        char *p;
 
-       if (verbose > 3) {
+       if (DEBUG_GTE(DEL, 3)) {
                rprintf(FINFO, "delete_dir_contents(%s) flags=%d\n",
                        fname, flags);
        }
@@ -284,7 +298,7 @@ static enum delret delete_dir_contents(char *fname, uint16 flags)
                struct file_struct *fp = dirlist->files[j];
 
                if (fp->flags & FLAG_MOUNT_DIR && S_ISDIR(fp->mode)) {
-                       if (verbose > 1) {
+                       if (DEBUG_GTE(DEL, 1)) {
                                rprintf(FINFO,
                                    "mount point, %s, pins parent directory\n",
                                    f_name(fp, NULL));
@@ -479,7 +493,7 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
                return;
        }
 
-       if (verbose > 2)
+       if (DEBUG_GTE(DEL, 2))
                rprintf(FINFO, "delete_in_dir(%s)\n", fbuf);
 
        if (allowed_lull)
@@ -516,12 +530,15 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
                if (!F_IS_ACTIVE(fp))
                        continue;
                if (fp->flags & FLAG_MOUNT_DIR && S_ISDIR(fp->mode)) {
-                       if (verbose > 1)
+                       if (INFO_GTE(MOUNT, 1))
                                rprintf(FINFO, "cannot delete mount point: %s\n",
                                        f_name(fp, NULL));
                        continue;
                }
-               if (flist_find(cur_flist, fp) < 0) {
+               /* Here we want to match regardless of file type.  Replacement
+                * of a file with one of another type is handled separately by
+                * a delete_item call with a DEL_MAKE_ROOM flag. */
+               if (flist_find_ignore_dirness(cur_flist, fp) < 0) {
                        int flags = DEL_RECURSE;
                        if (!(fp->mode & S_IWUSR) && !am_root && (uid_t)F_OWNER(fp) == our_uid)
                                flags |= DEL_NO_UID_WRITE;
@@ -557,11 +574,14 @@ static void do_delete_pass(void)
        for (j = 0; j < cur_flist->used; j++) {
                struct file_struct *file = cur_flist->sorted[j];
 
-               if (!(file->flags & FLAG_CONTENT_DIR))
+               f_name(file, fbuf);
+
+               if (!(file->flags & FLAG_CONTENT_DIR)) {
+                       change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(file));
                        continue;
+               }
 
-               f_name(file, fbuf);
-               if (verbose > 1 && file->flags & FLAG_TOP_DIR)
+               if (DEBUG_GTE(DEL, 1) && file->flags & FLAG_TOP_DIR)
                        rprintf(FINFO, "deleting in %s\n", fbuf);
 
                if (link_stat(fbuf, &st, keep_dirlinks) < 0
@@ -572,7 +592,7 @@ static void do_delete_pass(void)
        }
        delete_in_dir(NULL, NULL, &dev_zero);
 
-       if (do_progress && !am_server)
+       if (INFO_GTE(FLIST, 2) && !am_server)
                rprintf(FINFO, "                    \r");
 }
 
@@ -683,7 +703,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
        }
 
        iflags &= 0xffff;
-       if ((iflags & (SIGNIFICANT_ITEM_FLAGS|ITEM_REPORT_XATTR) || verbose > 1
+       if ((iflags & (SIGNIFICANT_ITEM_FLAGS|ITEM_REPORT_XATTR) || INFO_GTE(NAME, 2)
          || stdout_format_has_i > 1 || (xname && *xname)) && !read_batch) {
                if (protocol_version >= 29) {
                        if (ndx >= 0)
@@ -694,8 +714,11 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
                        if (iflags & ITEM_XNAME_FOLLOWS)
                                write_vstring(sock_f_out, xname, strlen(xname));
 #ifdef SUPPORT_XATTRS
-                       if (iflags & ITEM_REPORT_XATTR && !dry_run)
-                               send_xattr_request(NULL, file, sock_f_out);
+                       if (preserve_xattrs && !dry_run
+                        && iflags & (ITEM_REPORT_XATTR|ITEM_TRANSFER)) {
+                               send_xattr_request(NULL, file,
+                                       iflags & ITEM_REPORT_XATTR ? sock_f_out : -1);
+                       }
 #endif
                } else if (ndx >= 0) {
                        enum logcode code = logfile_format_has_i ? FINFO : FCLIENT;
@@ -750,18 +773,19 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
 {
        int32 blength;
        int s2length;
+       int64 l;
 
        if (block_size)
                blength = block_size;
        else if (len <= BLOCK_SIZE * BLOCK_SIZE)
                blength = BLOCK_SIZE;
        else {
+               int32 max_blength = protocol_version < 30 ? OLD_MAX_BLOCK_SIZE : MAX_BLOCK_SIZE;
                int32 c;
-               int64 l;
                int cnt;
                for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
-               if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
-                       blength = MAX_BLOCK_SIZE;
+               if (c < 0 || c >= max_blength)
+                       blength = max_blength;
                else {
                    blength = 0;
                    do {
@@ -780,7 +804,6 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
                s2length = SUM_LENGTH;
        } else {
                int32 c;
-               int64 l;
                int b = BLOCKSUM_BIAS;
                for (l = len; l >>= 1; b += 2) {}
                for (c = blength; (c >>= 1) && b; b--) {}
@@ -794,13 +817,16 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
        sum->blength    = blength;
        sum->s2length   = s2length;
        sum->remainder  = (int32)(len % blength);
-       sum->count      = (int32)(len / blength) + (sum->remainder != 0);
+       sum->count      = (int32)(l = (len / blength) + (sum->remainder != 0));
+
+       if ((int64)sum->count != l)
+               sum->count = -1;
 
-       if (sum->count && verbose > 2) {
+       if (sum->count && DEBUG_GTE(DELTASUM, 2)) {
                rprintf(FINFO,
-                       "count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
-                       (double)sum->count, (long)sum->remainder, (long)sum->blength,
-                       sum->s2length, (double)sum->flength);
+                       "count=%s rem=%ld blength=%ld s2length=%d flength=%s\n",
+                       big_num(sum->count), (long)sum->remainder, (long)sum->blength,
+                       sum->s2length, big_num(sum->flength));
        }
 }
 
@@ -810,7 +836,7 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
  *
  * Generate approximately one checksum every block_len bytes.
  */
-static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
+static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 {
        int32 i;
        struct map_struct *mapbuf;
@@ -818,10 +844,12 @@ static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
        OFF_T offset = 0;
 
        sum_sizes_sqroot(&sum, len);
+       if (sum.count < 0)
+               return -1;
        write_sum_head(f_out, &sum);
 
        if (append_mode > 0 && f_copy < 0)
-               return;
+               return 0;
 
        if (len > 0)
                mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
@@ -846,10 +874,10 @@ static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
                sum1 = get_checksum1(map, n1);
                get_checksum2(map, n1, sum2);
 
-               if (verbose > 3) {
+               if (DEBUG_GTE(DELTASUM, 3)) {
                        rprintf(FINFO,
-                               "chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
-                               (double)i, (double)offset - n1, (long)n1,
+                               "chunk[%s] offset=%s len=%ld sum1=%08lx\n",
+                               big_num(i), big_num(offset - n1), (long)n1,
                                (unsigned long)sum1);
                }
                write_int(f_out, sum1);
@@ -858,6 +886,8 @@ static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 
        if (mapbuf)
                unmap_file(mapbuf);
+
+       return 0;
 }
 
 
@@ -886,7 +916,7 @@ static int find_fuzzy(struct file_struct *file, struct file_list *dirlist)
 
                if (F_LENGTH(fp) == F_LENGTH(file)
                    && cmp_time(fp->modtime, file->modtime) == 0) {
-                       if (verbose > 4) {
+                       if (DEBUG_GTE(FUZZY, 2)) {
                                rprintf(FINFO,
                                        "fuzzy size/modtime match for %s\n",
                                        name);
@@ -901,7 +931,7 @@ static int find_fuzzy(struct file_struct *file, struct file_list *dirlist)
                /* Add some extra weight to how well the suffixes match. */
                dist += fuzzy_distance(suf, suf_len, fname_suf, fname_suf_len)
                      * 10;
-               if (verbose > 4) {
+               if (DEBUG_GTE(FUZZY, 2)) {
                        rprintf(FINFO, "fuzzy distance for %s = %d.%05d\n",
                                name, (int)(dist>>16), (int)(dist&0xFFFF));
                }
@@ -919,6 +949,7 @@ static int copy_altdest_file(const char *src, const char *dest, struct file_stru
 {
        char buf[MAXPATHLEN];
        const char *copy_to, *partialptr;
+       int save_preserve_xattrs = preserve_xattrs;
        int ok, fd_w;
 
        if (inplace) {
@@ -933,7 +964,7 @@ static int copy_altdest_file(const char *src, const char *dest, struct file_stru
        }
        cleanup_set(copy_to, NULL, NULL, -1, -1);
        if (copy_file(src, copy_to, fd_w, file->mode, 0) < 0) {
-               if (verbose) {
+               if (INFO_GTE(COPY, 1)) {
                        rsyserr(FINFO, errno, "copy_file %s => %s",
                                full_fname(src), copy_to);
                }
@@ -943,7 +974,9 @@ static int copy_altdest_file(const char *src, const char *dest, struct file_stru
                return -1;
        }
        partialptr = partial_dir ? partial_dir_fname(dest) : NULL;
+       preserve_xattrs = 0; /* xattrs were copied with file */
        ok = finish_transfer(dest, copy_to, src, partialptr, file, 1, 0);
+       preserve_xattrs = save_preserve_xattrs;
        cleanup_disable();
        return ok ? 0 : -1;
 }
@@ -1001,7 +1034,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
                                goto try_a_copy;
                        if (preserve_hard_links && F_IS_HLINKED(file))
                                finish_hard_link(file, fname, ndx, &sxp->st, itemizing, code, j);
-                       if (!maybe_ATTRS_REPORT && (verbose > 1 || stdout_format_has_i > 1)) {
+                       if (!maybe_ATTRS_REPORT && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) {
                                itemize(cmpbuf, file, ndx, 1, sxp,
                                        ITEM_LOCAL_CHANGE | ITEM_XNAME_FOLLOWS,
                                        0, "");
@@ -1010,7 +1043,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
 #endif
                if (itemizing)
                        itemize(cmpbuf, file, ndx, 0, sxp, 0, 0, NULL);
-               if (verbose > 1 && maybe_ATTRS_REPORT)
+               if (INFO_GTE(NAME, 2) && maybe_ATTRS_REPORT)
                        rprintf(FCLIENT, "%s is uptodate\n", fname);
                return -2;
        }
@@ -1024,8 +1057,8 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
                if (itemizing)
                        itemize(cmpbuf, file, ndx, 0, sxp, ITEM_LOCAL_CHANGE, 0, NULL);
                if (maybe_ATTRS_REPORT
-                && ((!itemizing && verbose && match_level == 2)
-                 || (verbose > 1 && match_level == 3))) {
+                && ((!itemizing && INFO_GTE(NAME, 1) && match_level == 2)
+                 || (INFO_GTE(NAME, 2) && match_level == 3))) {
                        code = match_level == 3 ? FCLIENT : FINFO;
                        rprintf(code, "%s%s\n", fname,
                                match_level == 3 ? " is uptodate" : "");
@@ -1165,13 +1198,13 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 #endif
                        match_level = 2;
                if (itemizing && stdout_format_has_i
-                && (verbose > 1 || stdout_format_has_i > 1)) {
+                && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) {
                        int chg = compare_dest && type != TYPE_DIR ? 0
                            : ITEM_LOCAL_CHANGE + (match_level == 3 ? ITEM_XNAME_FOLLOWS : 0);
                        char *lp = match_level == 3 ? "" : NULL;
                        itemize(cmpbuf, file, ndx, 0, sxp, chg + ITEM_MATCHED, 0, lp);
                }
-               if (verbose > 1 && maybe_ATTRS_REPORT) {
+               if (INFO_GTE(NAME, 2) && maybe_ATTRS_REPORT) {
                        rprintf(FCLIENT, "%s%s is uptodate\n",
                                fname, type == TYPE_DIR ? "/" : "");
                }
@@ -1184,7 +1217,8 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
 static void list_file_entry(struct file_struct *f)
 {
        char permbuf[PERMSTRING_SIZE];
-       double len;
+       int64 len;
+       int colwidth = human_readable ? 14 : 11;
 
        if (!F_IS_ACTIVE(f)) {
                /* this can happen if duplicate names were removed */
@@ -1198,15 +1232,16 @@ static void list_file_entry(struct file_struct *f)
 
 #ifdef SUPPORT_LINKS
        if (preserve_links && S_ISLNK(f->mode)) {
-               rprintf(FINFO, "%s %11.0f %s %s -> %s\n",
-                       permbuf, len, timestring(f->modtime),
-                       f_name(f, NULL), F_SYMLINK(f));
+               rprintf(FINFO, "%s %*s %s %s -> %s\n",
+                       permbuf, colwidth, comma_num(len),
+                       timestring(f->modtime), f_name(f, NULL),
+                       F_SYMLINK(f));
        } else
 #endif
        {
-               rprintf(FINFO, "%s %11.0f %s %s\n",
-                       permbuf, len, timestring(f->modtime),
-                       f_name(f, NULL));
+               rprintf(FINFO, "%s %*s %s %s\n",
+                       permbuf, colwidth, comma_num(len),
+                       timestring(f->modtime), f_name(f, NULL));
        }
 }
 
@@ -1257,7 +1292,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                   : inc_recurse && ndx != cur_flist->ndx_start - 1 ? -1
                   : 1;
 
-       if (verbose > 2)
+       if (DEBUG_GTE(GENR, 1))
                rprintf(FINFO, "recv_generator(%s,%d)\n", fname, ndx);
 
        if (list_only) {
@@ -1281,8 +1316,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                skip_dir = NULL;
        }
 
-       if (server_filter_list.head) {
-               if (check_filter(&server_filter_list, fname, is_dir) < 0) {
+       if (daemon_filter_list.head && (*fname != '.' || fname[1])) {
+               if (check_filter(&daemon_filter_list, FLOG, fname, is_dir) < 0) {
                        if (is_dir < 0)
                                return;
 #ifdef SUPPORT_HARD_LINKS
@@ -1361,7 +1396,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                else if (F_IS_HLINKED(file))
                        handle_skipped_hlink(file, itemizing, code, f_out);
 #endif
-               if (verbose > 1) {
+               if (INFO_GTE(SKIP, 1)) {
                        rprintf(FINFO, "not creating new %s \"%s\"\n",
                                is_dir ? "directory" : "file", fname);
                }
@@ -1374,7 +1409,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
        if (ignore_existing > 0 && statret == 0
         && (!is_dir || !S_ISDIR(sx.st.st_mode))) {
-               if (verbose > 1 && is_dir >= 0)
+               if (INFO_GTE(SKIP, 1) && is_dir >= 0)
                        rprintf(FINFO, "%s exists\n", fname);
 #ifdef SUPPORT_HARD_LINKS
                if (F_IS_HLINKED(file))
@@ -1429,6 +1464,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        if (j == -2) {
                                itemizing = 0;
                                code = FNONE;
+                               statret = 1;
                        } else if (j >= 0)
                                statret = 1;
                }
@@ -1451,8 +1487,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                goto cleanup;
                        }
                }
+#ifdef SUPPORT_XATTRS
+               if (preserve_xattrs && statret == 1)
+                       copy_xattrs(fnamecmpbuf, fname);
+#endif
                if (set_file_attrs(fname, file, real_ret ? NULL : &real_sx, NULL, 0)
-                   && verbose && code != FNONE && f_out != -1)
+                   && INFO_GTE(NAME, 1) && code != FNONE && f_out != -1)
                        rprintf(code, "%s/\n", fname);
 
                /* We need to ensure that the dirs in the transfer have writable
@@ -1480,8 +1520,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        }
                }
                else if (delete_during && f_out != -1 && !phase
-                   && BITS_SETnUNSET(file->flags, FLAG_CONTENT_DIR, FLAG_MISSING_DIR))
-                       delete_in_dir(fname, file, &real_sx.st.st_dev);
+                   && !(file->flags & FLAG_MISSING_DIR)) {
+                       if (file->flags & FLAG_CONTENT_DIR)
+                               delete_in_dir(fname, file, &real_sx.st.st_dev);
+                       else
+                               change_local_filter_dir(fname, strlen(fname), F_DEPTH(file));
+               }
                goto cleanup;
        }
 
@@ -1503,7 +1547,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 #ifdef SUPPORT_LINKS
                const char *sl = F_SYMLINK(file);
                if (safe_symlinks && unsafe_symlink(sl, fname)) {
-                       if (verbose) {
+                       if (INFO_GTE(NAME, 1)) {
                                if (solo_file)
                                        fname = f_name(file, NULL);
                                rprintf(FINFO,
@@ -1567,7 +1611,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                itemize(fname, file, ndx, statret, &sx,
                                        ITEM_LOCAL_CHANGE|ITEM_REPORT_CHANGE, 0, NULL);
                        }
-                       if (code != FNONE && verbose)
+                       if (code != FNONE && INFO_GTE(NAME, 1))
                                rprintf(code, "%s -> %s\n", fname, sl);
 #ifdef SUPPORT_HARD_LINKS
                        if (preserve_hard_links && F_IS_HLINKED(file))
@@ -1637,7 +1681,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        goto cleanup;
                }
 #endif
-               if (verbose > 2) {
+               if (DEBUG_GTE(GENR, 1)) {
                        rprintf(FINFO, "mknod(%s, 0%o, [%ld,%ld])\n",
                                fname, (int)file->mode,
                                (long)major(rdev), (long)minor(rdev));
@@ -1651,7 +1695,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                itemize(fname, file, ndx, statret, &sx,
                                        ITEM_LOCAL_CHANGE|ITEM_REPORT_CHANGE, 0, NULL);
                        }
-                       if (code != FNONE && verbose)
+                       if (code != FNONE && INFO_GTE(NAME, 1))
                                rprintf(code, "%s\n", fname);
 #ifdef SUPPORT_HARD_LINKS
                        if (preserve_hard_links && F_IS_HLINKED(file))
@@ -1671,7 +1715,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
        }
 
        if (max_size > 0 && F_LENGTH(file) > max_size) {
-               if (verbose > 1) {
+               if (INFO_GTE(SKIP, 1)) {
                        if (solo_file)
                                fname = f_name(file, NULL);
                        rprintf(FINFO, "%s is over max-size\n", fname);
@@ -1679,7 +1723,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                goto cleanup;
        }
        if (min_size > 0 && F_LENGTH(file) < min_size) {
-               if (verbose > 1) {
+               if (INFO_GTE(SKIP, 1)) {
                        if (solo_file)
                                fname = f_name(file, NULL);
                        rprintf(FINFO, "%s is under min-size\n", fname);
@@ -1689,7 +1733,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
        if (update_only > 0 && statret == 0
            && cmp_time(sx.st.st_mtime, file->modtime) > 0) {
-               if (verbose > 1)
+               if (INFO_GTE(SKIP, 1))
                        rprintf(FINFO, "%s is newer\n", fname);
 #ifdef SUPPORT_HARD_LINKS
                if (F_IS_HLINKED(file))
@@ -1739,7 +1783,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                if (j >= 0) {
                        fuzzy_file = fuzzy_dirlist->files[j];
                        f_name(fuzzy_file, fnamecmpbuf);
-                       if (verbose > 2) {
+                       if (DEBUG_GTE(FUZZY, 1)) {
                                rprintf(FINFO, "fuzzy basis selected for %s: %s\n",
                                        fname, fnamecmpbuf);
                        }
@@ -1861,25 +1905,31 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        close(fd);
                        goto cleanup;
                }
-               if ((f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0
-                && (errno != ENOENT || make_bak_dir(backupptr) < 0
-                 || (f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0)) {
-                       rsyserr(FERROR_XFER, errno, "open %s",
-                               full_fname(backupptr));
-                       unmake_file(back_file);
-                       back_file = NULL;
-                       close(fd);
-                       goto cleanup;
+               if ((f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
+                       int save_errno = errno ? errno : EINVAL; /* 0 paranoia */
+                       if (errno == ENOENT && make_bak_dir(backupptr) == 0) {
+                               if ((f_copy = do_open(backupptr, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0)
+                                       save_errno = errno ? errno : save_errno;
+                               else
+                                       save_errno = 0;
+                       }
+                       if (save_errno) {
+                               rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(backupptr));
+                               unmake_file(back_file);
+                               back_file = NULL;
+                               close(fd);
+                               goto cleanup;
+                       }
                }
                fnamecmp_type = FNAMECMP_BACKUP;
        }
 
-       if (verbose > 3) {
-               rprintf(FINFO, "gen mapped %s of size %.0f\n",
-                       fnamecmp, (double)sx.st.st_size);
+       if (DEBUG_GTE(DELTASUM, 3)) {
+               rprintf(FINFO, "gen mapped %s of size %s\n",
+                       fnamecmp, big_num(sx.st.st_size));
        }
 
-       if (verbose > 2)
+       if (DEBUG_GTE(DELTASUM, 2))
                rprintf(FINFO, "generating and sending sums for %d\n", ndx);
 
   notify_others:
@@ -1924,17 +1974,33 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
        if (statret != 0 || whole_file)
                write_sum_head(f_out, NULL);
-       else {
-               generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy);
+       else if (sx.st.st_size <= 0) {
+               write_sum_head(f_out, NULL);
+               close(fd);
+       } else {
+               if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) {
+                       rprintf(FWARNING,
+                           "WARNING: file is too large for checksum sending: %s\n",
+                           fnamecmp);
+                       write_sum_head(f_out, NULL);
+               }
                close(fd);
        }
 
   cleanup:
        if (back_file) {
+               int save_preserve_xattrs = preserve_xattrs;
                if (f_copy >= 0)
                        close(f_copy);
+#ifdef SUPPORT_XATTRS
+               if (preserve_xattrs) {
+                       copy_xattrs(fname, backupptr);
+                       preserve_xattrs = 0;
+               }
+#endif
                set_file_attrs(backupptr, back_file, NULL, NULL, 0);
-               if (verbose > 1) {
+               preserve_xattrs = save_preserve_xattrs;
+               if (INFO_GTE(BACKUP, 1)) {
                        rprintf(FINFO, "backed up %s to %s\n",
                                fname, backupptr);
                }
@@ -1994,7 +2060,7 @@ static void touch_up_dirs(struct file_list *flist, int ndx)
                if (!S_ISDIR(file->mode)
                 || (!implied_dirs && file->flags & FLAG_IMPLIED_DIR))
                        continue;
-               if (verbose > 3) {
+               if (DEBUG_GTE(TIME, 2)) {
                        fname = f_name(file, NULL);
                        rprintf(FINFO, "touch_up_dirs: %s (%d)\n",
                                NS(fname), i);
@@ -2011,10 +2077,13 @@ static void touch_up_dirs(struct file_list *flist, int ndx)
                         && cmp_time(st.st_mtime, file->modtime) != 0)
                                set_modtime(fname, file->modtime, file->mode);
                }
-               if (allowed_lull && !(counter % lull_mod))
-                       maybe_send_keepalive();
-               else if (!(counter & 0xFF))
-                       maybe_flush_socket(0);
+               if (counter >= loopchk_limit) {
+                       if (allowed_lull)
+                               maybe_send_keepalive();
+                       else
+                               maybe_flush_socket(0);
+                       counter = 0;
+               }
        }
 }
 
@@ -2028,8 +2097,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
        while (1) {
 #ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && (ndx = get_hlink_num()) != -1) {
-                       flist = flist_for_ndx(ndx);
-                       assert(flist != NULL);
+                       flist = flist_for_ndx(ndx, "check_for_finished_files.1");
                        file = flist->files[ndx - flist->ndx_start];
                        assert(file->flags & FLAG_HLINKED);
                        finish_hard_link(file, f_name(file, fbuf), ndx, NULL, itemizing, code, -1);
@@ -2052,7 +2120,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
                        ignore_times++;
 
                        flist = cur_flist;
-                       cur_flist = flist_for_ndx(ndx);
+                       cur_flist = flist_for_ndx(ndx, "check_for_finished_files.2");
 
                        file = cur_flist->files[ndx - cur_flist->ndx_start];
                        if (solo_file)
@@ -2101,11 +2169,12 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 
 void generate_files(int f_out, const char *local_name)
 {
-       int i, ndx;
+       int i, ndx, next_loopchk = 0;
        char fbuf[MAXPATHLEN];
        int itemizing;
        enum logcode code;
-       int save_do_progress = do_progress;
+       int save_info_flist = info_levels[INFO_FLIST];
+       int save_info_progress = info_levels[INFO_PROGRESS];
 
        if (protocol_version >= 29) {
                itemizing = 1;
@@ -2127,12 +2196,12 @@ void generate_files(int f_out, const char *local_name)
        solo_file = local_name;
        dir_tweaking = !(list_only || solo_file || dry_run);
        need_retouch_dir_times = preserve_times > 1;
-       lull_mod = allowed_lull * 5;
+       loopchk_limit = allowed_lull ? allowed_lull * 5 : 200;
        symlink_timeset_failed_flags = ITEM_REPORT_TIME
            | (protocol_version >= 30 || !am_server ? ITEM_REPORT_TIMEFAIL : 0);
        implied_dirs_are_missing = relative_paths && !implied_dirs && protocol_version < 30;
 
-       if (verbose > 2)
+       if (DEBUG_GTE(GENR, 1))
                rprintf(FINFO, "generator starting pid=%ld\n", (long)getpid());
 
        if (delete_before && !solo_file && cur_flist->used > 0)
@@ -2143,11 +2212,11 @@ void generate_files(int f_out, const char *local_name)
                if (!deldelay_buf)
                        out_of_memory("delete-delay");
        }
-       do_progress = 0;
+       info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0;
 
        if (append_mode > 0 || whole_file < 0)
                whole_file = 0;
-       if (verbose >= 2) {
+       if (DEBUG_GTE(FLIST, 1)) {
                rprintf(FINFO, "delta-transmission %s\n",
                        whole_file
                        ? "disabled for local transfer or --whole-file"
@@ -2172,19 +2241,24 @@ void generate_files(int f_out, const char *local_name)
 
                if (inc_recurse && cur_flist->parent_ndx >= 0) {
                        struct file_struct *fp = dir_flist->files[cur_flist->parent_ndx];
-                       f_name(fp, fbuf);
+                       if (solo_file)
+                               strlcpy(fbuf, solo_file, sizeof fbuf);
+                       else
+                               f_name(fp, fbuf);
                        ndx = cur_flist->ndx_start - 1;
                        recv_generator(fbuf, fp, ndx, itemizing, code, f_out);
-                       if (delete_during && dry_run < 2 && !list_only) {
-                               if (BITS_SETnUNSET(fp->flags, FLAG_CONTENT_DIR, FLAG_MISSING_DIR)) {
+                       if (delete_during && dry_run < 2 && !list_only
+                        && !(fp->flags & FLAG_MISSING_DIR)) {
+                               if (fp->flags & FLAG_CONTENT_DIR) {
                                        dev_t dirdev;
                                        if (one_file_system) {
                                                uint32 *devp = F_DIR_DEV_P(fp);
                                                dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
                                        } else
                                                dirdev = MAKEDEV(0, 0);
-                                       delete_in_dir(f_name(fp, fbuf), fp, &dirdev);
-                               }
+                                       delete_in_dir(fbuf, fp, &dirdev);
+                               } else
+                                       change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
                        }
                }
                for (i = cur_flist->low; i <= cur_flist->high; i++) {
@@ -2206,10 +2280,13 @@ void generate_files(int f_out, const char *local_name)
 
                        check_for_finished_files(itemizing, code, 0);
 
-                       if (allowed_lull && !(i % lull_mod))
-                               maybe_send_keepalive();
-                       else if (!(i & 0xFF))
-                               maybe_flush_socket(0);
+                       if (i + cur_flist->ndx_start >= next_loopchk) {
+                               if (allowed_lull)
+                                       maybe_send_keepalive();
+                               else
+                                       maybe_flush_socket(0);
+                               next_loopchk += loopchk_limit;
+                       }
                }
 
                if (!inc_recurse) {
@@ -2225,10 +2302,13 @@ void generate_files(int f_out, const char *local_name)
                }
        } while ((cur_flist = cur_flist->next) != NULL);
 
+       if (read_batch && inc_recurse)
+               write_ndx(f_out, NDX_DONE);
+
        if (delete_during)
                delete_in_dir(NULL, NULL, &dev_zero);
        phase++;
-       if (verbose > 2)
+       if (DEBUG_GTE(GENR, 1))
                rprintf(FINFO, "generate_files phase=%d\n", phase);
 
        while (1) {
@@ -2239,10 +2319,12 @@ void generate_files(int f_out, const char *local_name)
        }
 
        phase++;
-       if (verbose > 2)
+       if (DEBUG_GTE(GENR, 1))
                rprintf(FINFO, "generate_files phase=%d\n", phase);
 
        write_ndx(f_out, NDX_DONE);
+       write_del_stats(f_out);
+
        /* Reduce round-trip lag-time for a useless delay-updates phase. */
        if (protocol_version >= 29 && !delay_updates)
                write_ndx(f_out, NDX_DONE);
@@ -2257,7 +2339,7 @@ void generate_files(int f_out, const char *local_name)
 
        if (protocol_version >= 29) {
                phase++;
-               if (verbose > 2)
+               if (DEBUG_GTE(GENR, 1))
                        rprintf(FINFO, "generate_files phase=%d\n", phase);
                if (delay_updates)
                        write_ndx(f_out, NDX_DONE);
@@ -2266,7 +2348,9 @@ void generate_files(int f_out, const char *local_name)
                        wait_for_receiver();
        }
 
-       do_progress = save_do_progress;
+       info_levels[INFO_FLIST] = save_info_flist;
+       info_levels[INFO_PROGRESS] = save_info_progress;
+
        if (delete_during == 2)
                do_delayed_deletions(fbuf);
        if (delete_after && !solo_file && file_total > 0)
@@ -2276,13 +2360,13 @@ void generate_files(int f_out, const char *local_name)
         && dir_tweaking && (!inc_recurse || delete_during == 2))
                touch_up_dirs(dir_flist, -1);
 
-       if (max_delete >= 0 && deletion_count > max_delete) {
-               rprintf(FINFO,
+       if (max_delete >= 0 && skipped_deletes) {
+               rprintf(FWARNING,
                        "Deletions stopped due to --max-delete limit (%d skipped)\n",
-                       deletion_count - max_delete);
+                       skipped_deletes);
                io_error |= IOERR_DEL_LIMIT;
        }
 
-       if (verbose > 2)
+       if (DEBUG_GTE(GENR, 1))
                rprintf(FINFO, "generate_files finished\n");
 }