Applying my updated version of Craig Barratt's buffered I/O patch.
[rsync/rsync.git] / flist.c
diff --git a/flist.c b/flist.c
index f6ae224..f905fb8 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -57,8 +57,7 @@ extern int relative_paths;
 extern int implied_dirs;
 extern int copy_links;
 extern int copy_unsafe_links;
-extern int remote_version;
-extern int io_error;
+extern int protocol_version;
 extern int sanitize_paths;
 
 extern int read_batch;
@@ -68,6 +67,8 @@ extern struct exclude_struct **exclude_list;
 extern struct exclude_struct **server_exclude_list;
 extern struct exclude_struct **local_exclude_list;
 
+int io_error;
+
 static struct file_struct null_file;
 
 static void clean_flist(struct file_list *flist, int strip_root, int no_dups);
@@ -123,10 +124,10 @@ static struct string_area *string_area_new(int size)
 
        if (size <= 0)
                size = ARENA_SIZE;
-       a = malloc(sizeof(*a));
+       a = new(struct string_area);
        if (!a)
                out_of_memory("string_area_new");
-       a->current = a->base = malloc(size);
+       a->current = a->base = new_array(char, size);
        if (!a->current)
                out_of_memory("string_area_new buffer");
        a->end = a->base + size;
@@ -262,19 +263,26 @@ static int check_exclude_file(char *fname, int is_dir, int exclude_level)
        if (exclude_level == NO_EXCLUDES)
                return 0;
 #endif
-       if (fname && fname[0] == '.' && !fname[1]) {
+       if (fname) {
                /* never exclude '.', even if somebody does --exclude '*' */
-               return 0;
+               if (fname[0] == '.' && !fname[1])
+                       return 0;
+               /* Handle the -R version of the '.' dir. */
+               if (fname[0] == '/') {
+                       int len = strlen(fname);
+                       if (fname[len-1] == '.' && fname[len-2] == '/')
+                               return 0;
+               }
        }
-       if (server_exclude_list &&
-           check_exclude(server_exclude_list, fname, is_dir))
+       if (server_exclude_list
+        && check_exclude(server_exclude_list, fname, is_dir))
                return 1;
        if (exclude_level != ALL_EXCLUDES)
                return 0;
        if (exclude_list && check_exclude(exclude_list, fname, is_dir))
                return 1;
-       if (local_exclude_list &&
-           check_exclude(local_exclude_list, fname, is_dir))
+       if (local_exclude_list
+        && check_exclude(local_exclude_list, fname, is_dir))
                return 1;
        return 0;
 }
@@ -320,7 +328,6 @@ static char *flist_dir;
 static void flist_expand(struct file_list *flist)
 {
        if (flist->count >= flist->malloced) {
-               size_t new_bytes;
                void *new_ptr;
 
                if (flist->malloced < 1000)
@@ -328,16 +335,19 @@ static void flist_expand(struct file_list *flist)
                else
                        flist->malloced *= 2;
 
-               new_bytes = sizeof(flist->files[0]) * flist->malloced;
-
-               if (flist->files)
-                       new_ptr = realloc(flist->files, new_bytes);
-               else
-                       new_ptr = malloc(new_bytes);
+               if (flist->files) {
+                       new_ptr = realloc_array(flist->files,
+                                               struct file_struct *,
+                                               flist->malloced);
+               } else {
+                       new_ptr = new_array(struct file_struct *,
+                                           flist->malloced);
+               }
 
                if (verbose >= 2) {
                        rprintf(FINFO, "expand file_list to %.0f bytes, did%s move\n",
-                               (double) new_bytes,
+                               (double)sizeof(flist->files[0])
+                               * flist->malloced,
                                (new_ptr == flist->files) ? " not" : "");
                }
 
@@ -359,7 +369,7 @@ static void send_file_entry(struct file_struct *file, int f,
        static uid_t last_uid;
        static gid_t last_gid;
        static char lastname[MAXPATHLEN];
-       char *fname;
+       char *fname, buf[MAXPATHLEN];
        int l1, l2;
 
        if (f == -1)
@@ -372,7 +382,7 @@ static void send_file_entry(struct file_struct *file, int f,
 
        io_write_phase = "send_file_entry";
 
-       fname = f_name(file);
+       fname = f_name_to(file, buf, sizeof buf);
 
        flags = base_flags;
 
@@ -439,7 +449,7 @@ static void send_file_entry(struct file_struct *file, int f,
 
 #if SUPPORT_HARD_LINKS
        if (preserve_hard_links && S_ISREG(file->mode)) {
-               if (remote_version < 26) {
+               if (protocol_version < 26) {
                        /* 32-bit dev_t and ino_t */
                        write_int(f, (int) file->dev);
                        write_int(f, (int) file->inode);
@@ -452,7 +462,7 @@ static void send_file_entry(struct file_struct *file, int f,
 #endif
 
        if (always_checksum) {
-               if (remote_version < 21) {
+               if (protocol_version < 21) {
                        write_buf(f, file->sum, 2);
                } else {
                        write_buf(f, file->sum, MD4_SUM_LENGTH);
@@ -495,7 +505,7 @@ static void receive_file_entry(struct file_struct **fptr,
        else
                l2 = read_byte(f);
 
-       file = (struct file_struct *) malloc(sizeof(*file));
+       file = new(struct file_struct);
        if (!file)
                out_of_memory("receive_file_entry");
        memset((char *) file, 0, sizeof(*file));
@@ -562,7 +572,7 @@ static void receive_file_entry(struct file_struct **fptr,
                        rprintf(FERROR, "overflow: l=%d\n", l);
                        overflow("receive_file_entry");
                }
-               file->link = (char *) malloc(l + 1);
+               file->link = new_array(char, l + 1);
                if (!file->link)
                        out_of_memory("receive_file_entry 2");
                read_sbuf(f, file->link, l);
@@ -572,7 +582,7 @@ static void receive_file_entry(struct file_struct **fptr,
        }
 #if SUPPORT_HARD_LINKS
        if (preserve_hard_links && S_ISREG(file->mode)) {
-               if (remote_version < 26) {
+               if (protocol_version < 26) {
                        file->dev = read_int(f);
                        file->inode = read_int(f);
                } else {
@@ -583,10 +593,10 @@ static void receive_file_entry(struct file_struct **fptr,
 #endif
 
        if (always_checksum) {
-               file->sum = (char *) malloc(MD4_SUM_LENGTH);
+               file->sum = new_array(char, MD4_SUM_LENGTH);
                if (!file->sum)
                        out_of_memory("md4 sum");
-               if (remote_version < 21) {
+               if (protocol_version < 21) {
                        read_buf(f, file->sum, 2);
                } else {
                        read_buf(f, file->sum, MD4_SUM_LENGTH);
@@ -685,9 +695,9 @@ struct file_struct *make_file(char *fname, struct string_area **ap,
                                return NULL;
                        }
                }
-               io_error = 1;
-               rprintf(FERROR, "readlink %s: %s\n",
-                       fname, strerror(save_errno));
+               io_error |= IOERR_GENERAL;
+               rprintf(FERROR, "readlink %s failed: %s\n",
+                       full_fname(fname), strerror(save_errno));
                return NULL;
        }
 
@@ -716,7 +726,7 @@ struct file_struct *make_file(char *fname, struct string_area **ap,
        if (verbose > 2)
                rprintf(FINFO, "make_file(%s,*,%d)\n", fname, exclude_level);
 
-       file = (struct file_struct *) malloc(sizeof(*file));
+       file = new(struct file_struct);
        if (!file)
                out_of_memory("make_file");
        memset((char *) file, 0, sizeof(*file));
@@ -787,11 +797,11 @@ struct file_struct *make_file(char *fname, struct string_area **ap,
 }
 
 
-
 void send_file_name(int f, struct file_list *flist, char *fname,
                    int recursive, unsigned base_flags)
 {
        struct file_struct *file;
+       char buf[MAXPATHLEN];
        extern int delete_excluded;
 
        /* f is set to -1 when calculating deletion file list */
@@ -806,7 +816,7 @@ void send_file_name(int f, struct file_list *flist, char *fname,
 
        flist_expand(flist);
 
-       if (write_batch)        /*  dw  */
+       if (write_batch)
                file->flags = FLAG_DELETE;
 
        if (file->basename[0]) {
@@ -817,14 +827,13 @@ void send_file_name(int f, struct file_list *flist, char *fname,
        if (S_ISDIR(file->mode) && recursive) {
                struct exclude_struct **last_exclude_list =
                    local_exclude_list;
-               send_directory(f, flist, f_name(file));
+               send_directory(f, flist, f_name_to(file, buf, sizeof buf));
                local_exclude_list = last_exclude_list;
                return;
        }
 }
 
 
-
 static void send_directory(int f, struct file_list *flist, char *dir)
 {
        DIR *d;
@@ -835,8 +844,9 @@ static void send_directory(int f, struct file_list *flist, char *dir)
 
        d = opendir(dir);
        if (!d) {
-               io_error = 1;
-               rprintf(FERROR, "opendir(%s): %s\n", dir, strerror(errno));
+               io_error |= IOERR_GENERAL;
+               rprintf(FERROR, "opendir %s failed: %s\n",
+                       full_fname(dir), strerror(errno));
                return;
        }
 
@@ -844,10 +854,9 @@ static void send_directory(int f, struct file_list *flist, char *dir)
        l = strlen(fname);
        if (fname[l - 1] != '/') {
                if (l == MAXPATHLEN - 1) {
-                       io_error = 1;
-                       rprintf(FERROR,
-                               "skipping long-named directory %s\n",
-                               fname);
+                       io_error |= IOERR_GENERAL;
+                       rprintf(FERROR, "skipping long-named directory: %s\n",
+                               full_fname(fname));
                        closedir(d);
                        return;
                }
@@ -863,21 +872,26 @@ static void send_directory(int f, struct file_list *flist, char *dir)
                        strcpy(p, ".cvsignore");
                        add_exclude_file(&exclude_list,fname,MISSING_OK,ADD_EXCLUDE);
                } else {
-                       io_error = 1;
+                       io_error |= IOERR_GENERAL;
                        rprintf(FINFO,
                                "cannot cvs-exclude in long-named directory %s\n",
-                               fname);
+                               full_fname(fname));
                }
        }
 
-       for (di = readdir(d); di; di = readdir(d)) {
+       for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) {
                char *dname = d_name(di);
-               if (dname[0] == '.' && (dname[1] == '\0' ||
-                   (dname[1] == '.' && dname[2] == '\0')))
+               if (dname[0] == '.' && (dname[1] == '\0'
+                   || (dname[1] == '.' && dname[2] == '\0')))
                        continue;
                strlcpy(p, dname, MAXPATHLEN - l);
                send_file_name(f, flist, fname, recurse, 0);
        }
+       if (errno) {
+               io_error |= IOERR_GENERAL;
+               rprintf(FERROR, "readdir(%s): (%d) %s\n",
+                   dir, errno, strerror(errno));
+       }
 
        if (local_exclude_list)
                free_exclude_list(&local_exclude_list); /* Zeros pointer too */
@@ -910,11 +924,11 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
        flist = flist_new();
 
        if (f != -1) {
-               io_start_buffering(f);
+               io_start_buffering_out(f);
                if (filesfrom_fd >= 0) {
                        if (argv[0] && !push_dir(argv[0], 0)) {
-                               rprintf(FERROR, "push_dir %s : %s\n",
-                                       argv[0], strerror(errno));
+                               rprintf(FERROR, "push_dir %s failed: %s\n",
+                                       full_fname(argv[0]), strerror(errno));
                                exit_cleanup(RERR_FILESELECT);
                        }
                        use_ff_fd = 1;
@@ -938,14 +952,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                }
 
                l = strlen(fname);
-               if (l != 1 && fname[l - 1] == '/') {
-                       if ((l == 2) && (fname[0] == '.')) {
-                               /*  Turn ./ into just . rather than ./.
-                                  This was put in to avoid a problem with
-                                  rsync -aR --delete from ./
-                                  The send_file_name() below of ./ was
-                                  mysteriously preventing deletes */
-                               fname[1] = 0;
+               if (fname[l - 1] == '/') {
+                       if (l == 2 && fname[0] == '.') {
+                               /* Turn "./" into just "." rather than "./." */
+                               fname[1] = '\0';
                        } else {
                                strlcat(fname, ".", MAXPATHLEN);
                        }
@@ -953,9 +963,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
                if (link_stat(fname, &st) != 0) {
                        if (f != -1) {
-                               io_error = 1;
-                               rprintf(FERROR, "link_stat %s : %s\n",
-                                       fname, strerror(errno));
+                               io_error |= IOERR_GENERAL;
+                               rprintf(FERROR, "link_stat %s failed: %s\n",
+                                       full_fname(fname), strerror(errno));
                        }
                        continue;
                }
@@ -1020,9 +1030,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                        olddir = push_dir(dir, 1);
 
                        if (!olddir) {
-                               io_error = 1;
-                               rprintf(FERROR, "push_dir %s : %s\n",
-                                       dir, strerror(errno));
+                               io_error |= IOERR_GENERAL;
+                               rprintf(FERROR, "push_dir %s failed: %s\n",
+                                       full_fname(dir), strerror(errno));
                                continue;
                        }
 
@@ -1037,8 +1047,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                if (olddir != NULL) {
                        flist_dir = NULL;
                        if (pop_dir(olddir) != 0) {
-                               rprintf(FERROR, "pop_dir %s : %s\n",
-                                       dir, strerror(errno));
+                               rprintf(FERROR, "pop_dir %s failed: %s\n",
+                                       full_fname(dir), strerror(errno));
                                exit_cleanup(RERR_FILESELECT);
                        }
                }
@@ -1070,7 +1080,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
                io_end_buffering();
                stats.flist_size = stats.total_written - start_write;
                stats.num_files = flist->count;
-               if (write_batch)        /*  dw  */
+               if (write_batch)
                        write_batch_flist_info(flist->count, flist->files);
        }
 
@@ -1093,15 +1103,13 @@ struct file_list *recv_file_list(int f)
 
        start_read = stats.total_read;
 
-       flist = (struct file_list *) malloc(sizeof(flist[0]));
+       flist = new(struct file_list);
        if (!flist)
                goto oom;
 
        flist->count = 0;
        flist->malloced = 1000;
-       flist->files =
-           (struct file_struct **) malloc(sizeof(flist->files[0]) *
-                                          flist->malloced);
+       flist->files = new_array(struct file_struct *, flist->malloced);
        if (!flist->files)
                goto oom;
 
@@ -1120,9 +1128,10 @@ struct file_list *recv_file_list(int f)
 
                maybe_emit_filelist_progress(flist);
 
-               if (verbose > 2)
+               if (verbose > 2) {
                        rprintf(FINFO, "recv_file_name(%s)\n",
                                f_name(flist->files[i]));
+               }
        }
 
 
@@ -1173,10 +1182,6 @@ struct file_list *recv_file_list(int f)
 }
 
 
-/*
- * XXX: This is currently the hottest function while building the file
- * list, because building f_name()s every time is expensive.
- **/
 int file_compare(struct file_struct **f1, struct file_struct **f2)
 {
        if (!(*f1)->basename && !(*f2)->basename)
@@ -1187,7 +1192,7 @@ int file_compare(struct file_struct **f1, struct file_struct **f2)
                return 1;
        if ((*f1)->dirname == (*f2)->dirname)
                return u_strcmp((*f1)->basename, (*f2)->basename);
-       return u_strcmp(f_name(*f1), f_name(*f2));
+       return f_name_cmp(*f1, *f2);
 }
 
 
@@ -1243,7 +1248,7 @@ struct file_list *flist_new(void)
 {
        struct file_list *flist;
 
-       flist = (struct file_list *) malloc(sizeof(flist[0]));
+       flist = new(struct file_list);
        if (!flist)
                out_of_memory("send_file_list");
 
@@ -1290,8 +1295,7 @@ void flist_free(struct file_list *flist)
  */
 static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 {
-       int i;
-       char *name, *prev_name = NULL;
+       int i, prev_i = 0;
 
        if (!flist || flist->count == 0)
                return;
@@ -1301,20 +1305,24 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 
        for (i = no_dups? 0 : flist->count; i < flist->count; i++) {
                if (flist->files[i]->basename) {
-                       prev_name = f_name(flist->files[i]);
+                       prev_i = i;
                        break;
                }
        }
        while (++i < flist->count) {
                if (!flist->files[i]->basename)
                        continue;
-               name = f_name(flist->files[i]);
-               if (strcmp(name, prev_name) == 0) {
+               if (f_name_cmp(flist->files[i], flist->files[prev_i]) == 0) {
                        if (verbose > 1 && !am_server) {
                                rprintf(FINFO,
                                        "removing duplicate name %s from file list %d\n",
-                                       name, i);
+                                       f_name(flist->files[i]), i);
                        }
+                       /* Make sure that if we unduplicate '.', that we don't
+                        * lose track of a user-specified starting point (or
+                        * else deletions will mysteriously fail with -R). */
+                       if (flist->files[i]->flags & FLAG_DELETE)
+                               flist->files[prev_i]->flags |= FLAG_DELETE;
                        /* it's not great that the flist knows the semantics of
                         * the file memory usage, but i'd rather not add a flag
                         * byte to that struct.
@@ -1324,7 +1332,8 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
                        else
                                free_file(flist->files[i]);
                }
-               prev_name = name;
+               else
+                       prev_i = i;
        }
 
        if (strip_root) {
@@ -1360,33 +1369,100 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 }
 
 
-/*
- * return the full filename of a flist entry
- *
- * This function is too expensive at the moment, because it copies
- * strings when often we only want to compare them.  In any case,
- * using strlcat is silly because it will walk the string repeatedly.
+enum fnc_state { fnc_DIR, fnc_SLASH, fnc_BASE };
+
+/* Compare the names of two file_struct entities, just like strcmp()
+ * would do if it were operating on the joined strings.  We assume
+ * that there are no 0-length strings.
  */
-char *f_name(struct file_struct *f)
+int f_name_cmp(struct file_struct *f1, struct file_struct *f2)
 {
-       static char names[10][MAXPATHLEN];
-       static int n;
-       char *p = names[n];
+       int dif;
+       const uchar *c1, *c2;
+       enum fnc_state state1 = fnc_DIR, state2 = fnc_DIR;
+
+       if (!f1 || !f1->basename) {
+               if (!f2 || !f2->basename)
+                       return 0;
+               return -1;
+       }
+       if (!f2 || !f2->basename)
+               return 1;
 
+       if (!(c1 = f1->dirname)) {
+               state1 = fnc_BASE;
+               c1 = f1->basename;
+       }
+       if (!(c2 = f2->dirname)) {
+               state2 = fnc_BASE;
+               c2 = f2->basename;
+       }
+
+       while (1) {
+               if ((dif = (int)*c1 - (int)*c2) != 0)
+                       break;
+               if (!*++c1) {
+                       switch (state1) {
+                       case fnc_DIR:
+                               state1 = fnc_SLASH;
+                               c1 = "/";
+                               break;
+                       case fnc_SLASH:
+                               state1 = fnc_BASE;
+                               c1 = f1->basename;
+                               break;
+                       case fnc_BASE:
+                               break;
+                       }
+               }
+               if (!*++c2) {
+                       switch (state2) {
+                       case fnc_DIR:
+                               state2 = fnc_SLASH;
+                               c2 = "/";
+                               break;
+                       case fnc_SLASH:
+                               state2 = fnc_BASE;
+                               c2 = f2->basename;
+                               break;
+                       case fnc_BASE:
+                               if (!*c1)
+                                       return 0;
+                               break;
+                       }
+               }
+       }
+
+       return dif;
+}
+
+
+/* Return a copy of the full filename of a flist entry, using the indicated
+ * buffer.
+ */
+char *f_name_to(struct file_struct *f, char *buf, int bsize)
+{
        if (!f || !f->basename)
                return NULL;
 
-       n = (n + 1) % 10;
-
        if (f->dirname) {
-               int off;
+               int off = strlcpy(buf, f->dirname, bsize);
+               off += strlcpy(buf + off, "/", bsize - off);
+               strlcpy(buf + off, f->basename, bsize - off);
+       } else
+               strlcpy(buf, f->basename, bsize);
+       return buf;
+}
 
-               off = strlcpy(p, f->dirname, MAXPATHLEN);
-               off += strlcpy(p + off, "/", MAXPATHLEN - off);
-               off += strlcpy(p + off, f->basename, MAXPATHLEN - off);
-       } else {
-               strlcpy(p, f->basename, MAXPATHLEN);
-       }
 
-       return p;
+/* Like f_name_to(), but we rotate through 5 static buffers of our own.
+ */
+char *f_name(struct file_struct *f)
+{
+       static char names[5][MAXPATHLEN];
+       static unsigned int n;
+
+       n = (n + 1) % (sizeof names / sizeof names[0]);
+
+       return f_name_to(f, names[n], sizeof names[0]);
 }