- Revamped the hard-link algorithm to save memory.
[rsync/rsync.git] / generator.c
index 71bd5a7..776b282 100644 (file)
@@ -97,7 +97,10 @@ int ignore_perishable = 0;
 int non_perishable_cnt = 0;
 
 static int deletion_count = 0; /* used to implement --max-delete */
-static FILE *delete_delay_fp = NULL;
+static int deldelay_size = 0, deldelay_cnt = 0;
+static char *deldelay_buf = NULL;
+static int deldelay_fd = -1;
+static BOOL solo_file = 0;
 
 /* For calling delete_item() and delete_dir_contents(). */
 #define DEL_RECURSE            (1<<1) /* recurse */
@@ -266,67 +269,135 @@ static enum delret delete_dir_contents(char *fname, int flags)
        return ret;
 }
 
-static void start_delete_temp(void)
+static int start_delete_delay_temp(void)
 {
        char fnametmp[MAXPATHLEN];
-       int fd, save_dry_run = dry_run;
+       int save_dry_run = dry_run;
 
        dry_run = 0;
        if (!get_tmpname(fnametmp, "deldelay")
-        || (fd = do_mkstemp(fnametmp, 0600)) < 0
-        || !(delete_delay_fp = fdopen(fd, "w+"))) {
-               rprintf(FERROR, "Unable to create delete-delay temp file.\n");
-               exit_cleanup(RERR_FILEIO);
+        || (deldelay_fd = do_mkstemp(fnametmp, 0600)) < 0) {
+               rprintf(FINFO, "NOTE: Unable to create delete-delay temp file--"
+                       "switching to --delete-after.\n");
+               delete_during = 0;
+               delete_after = 1;
+               dry_run = save_dry_run;
+               return 0;
        }
-       dry_run = save_dry_run;
        unlink(fnametmp);
+       dry_run = save_dry_run;
+       return 1;
 }
 
-static int read_delay_line(FILE *fp, char *buf, int bsize)
+static int flush_delete_delay(void)
 {
-       int ch, mode = 0;
+       if (write(deldelay_fd, deldelay_buf, deldelay_cnt) != deldelay_cnt) {
+               rsyserr(FERROR, errno, "flush of delete-delay buffer");
+               delete_during = 0;
+               delete_after = 1;
+               close(deldelay_fd);
+               return 0;
+       }
+       deldelay_cnt = 0;
+       return 1;
+}
 
-       if ((ch = fgetc(fp)) == EOF)
-               return -1;
+static int remember_delete(struct file_struct *file, const char *fname)
+{
+       int len;
+       
+       while (1) {
+               len = snprintf(deldelay_buf + deldelay_cnt,
+                              deldelay_size - deldelay_cnt,
+                              "%x %s%c", (int)file->mode, fname, '\0');
+               if ((deldelay_cnt += len) <= deldelay_size)
+                       break;
+               if (deldelay_fd < 0 && !start_delete_delay_temp())
+                       return 0;
+               deldelay_cnt -= len;
+               if (!flush_delete_delay())
+                       return 0;
+       }
+
+       return 1;
+}
+
+static int read_delay_line(char *buf)
+{
+       static int read_pos = 0;
+       int j, len, mode;
+       char *bp, *past_space;
 
        while (1) {
-               if (ch == ' ')
+               for (j = read_pos; j < deldelay_cnt && deldelay_buf[j]; j++) {}
+               if (j < deldelay_cnt)
                        break;
-               if (ch > '7' || ch < '0') {
-                       rprintf(FERROR, "invalid data in delete-delay file.\n");
-                       exit_cleanup(RERR_FILEIO);
+               if (deldelay_fd < 0) {
+                       if (j > read_pos)
+                               goto invalid_data;
+                       return -1;
+               }
+               deldelay_cnt -= read_pos;
+               if (deldelay_cnt == deldelay_size)
+                       goto invalid_data;
+               if (deldelay_cnt && read_pos) {
+                       memmove(deldelay_buf, deldelay_buf + read_pos,
+                               deldelay_cnt);
+               }
+               len = read(deldelay_fd, deldelay_buf + deldelay_cnt,
+                          deldelay_size - deldelay_cnt);
+               if (len == 0) {
+                       if (deldelay_cnt) {
+                               rprintf(FERROR,
+                                   "ERROR: unexpected EOF in delete-delay file.\n");
+                       }
+                       return -1;
                }
-               mode = mode*8 + ch - '0';
-               if ((ch = fgetc(fp)) == EOF) {
-                 unexpected_eof:
-                       rprintf(FERROR, "unexpected EOF in delete-delay file.\n");
-                       exit_cleanup(RERR_FILEIO);
+               if (len < 0) {
+                       rsyserr(FERROR, errno,
+                               "reading delete-delay file");
+                       return -1;
                }
+               deldelay_cnt += len;
+               read_pos = 0;
        }
 
-       while (1) {
-               if ((ch = fgetc(fp)) == EOF)
-                       goto unexpected_eof;
-               if (bsize-- <= 0) {
-                       rprintf(FERROR, "filename too long in delete-delay file.\n");
-                       exit_cleanup(RERR_FILEIO);
-               }
-               *buf++ = (char)ch;
-               if (ch == '\0')
-                       break;
+       bp = deldelay_buf + read_pos;
+
+       if (sscanf(bp, "%x ", &mode) != 1) {
+         invalid_data:
+               rprintf(FERROR, "ERROR: invalid data in delete-delay file.\n");
+               return -1;
        }
+       past_space = strchr(bp, ' ') + 1;
+       len = j - read_pos - (past_space - bp) + 1; /* count the '\0' */
+       read_pos = j + 1;
+
+       if (len > MAXPATHLEN) {
+               rprintf(FERROR, "ERROR: filename too long in delete-delay file.\n");
+               return -1;
+       }
+
+       /* The caller needs the name in a MAXPATHLEN buffer, so we copy it
+        * instead of returning a pointer to our buffer. */
+       memcpy(buf, past_space, len);
 
        return mode;
 }
 
-static void delayed_deletions(char *delbuf)
+static void do_delayed_deletions(char *delbuf)
 {
        int mode;
 
-       fseek(delete_delay_fp, 0, 0);
-       while ((mode = read_delay_line(delete_delay_fp, delbuf, MAXPATHLEN)) >= 0)
+       if (deldelay_fd >= 0) {
+               if (deldelay_cnt && !flush_delete_delay())
+                       return;
+               lseek(deldelay_fd, 0, 0);
+       }
+       while ((mode = read_delay_line(delbuf)) >= 0)
                delete_item(delbuf, mode, NULL, DEL_RECURSE);
-       fclose(delete_delay_fp);
+       if (deldelay_fd >= 0)
+               close(deldelay_fd);
 }
 
 /* This function is used to implement per-directory deletion, and is used by
@@ -401,9 +472,10 @@ static void delete_in_dir(struct file_list *flist, char *fbuf,
                }
                if (flist_find(flist, fp) < 0) {
                        f_name(fp, delbuf);
-                       if (delete_delay_fp)
-                               fprintf(delete_delay_fp, "%o %s%c", (short)fp->mode, delbuf, '\0');
-                       else
+                       if (delete_during == 2) {
+                               if (!remember_delete(fp, delbuf))
+                                       break;
+                       } else
                                delete_item(delbuf, fp->mode, NULL, DEL_RECURSE);
                }
        }
@@ -448,7 +520,7 @@ static void do_delete_pass(struct file_list *flist)
 int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
 {
        if (preserve_perms
-        && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
+        && (unsigned)(st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
                return 0;
 
        if (am_root && preserve_uid && st->st_uid != F_UID(file))
@@ -475,7 +547,7 @@ void itemize(struct file_struct *file, int ndx, int statret, STRUCT_STAT *st,
                  && (!(iflags & ITEM_XNAME_FOLLOWS) || *xname))
                 || (keep_time && cmp_time(file->modtime, st->st_mtime) != 0))
                        iflags |= ITEM_REPORT_TIME;
-               if ((file->mode & CHMOD_BITS) != (st->st_mode & CHMOD_BITS))
+               if ((unsigned)(st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
                        iflags |= ITEM_REPORT_PERMS;
                if (preserve_uid && am_root && F_UID(file) != st->st_uid)
                        iflags |= ITEM_REPORT_OWNER;
@@ -712,6 +784,7 @@ static int find_fuzzy(struct file_struct *file, struct file_list *dirlist)
        return lowest_j;
 }
 
+#ifdef SUPPORT_HARD_LINKS
 void check_for_finished_hlinks(int itemizing, enum logcode code)
 {
        struct file_struct *file;
@@ -728,6 +801,7 @@ void check_for_finished_hlinks(int itemizing, enum logcode code)
                hard_link_cluster(file, ndx, itemizing, code, -1);
        }
 }
+#endif
 
 /* This is only called for regular files.  We return -2 if we've finished
  * handling the file, -1 if no dest-linking occurred, or a non-negative
@@ -816,8 +890,10 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
                        rprintf(code, "%s%s\n", fname,
                                match_level == 3 ? " is uptodate" : "");
                }
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file))
                        hard_link_cluster(file, ndx, itemizing, code, j);
+#endif
                return -2;
        }
 
@@ -835,6 +911,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
        int best_match = -1;
        int match_level = 0;
        enum nonregtype type;
+       uint32 *devp;
        int len, j = 0;
 
 #ifndef SUPPORT_LINKS
@@ -891,7 +968,8 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx,
                        break;
                case TYPE_SPECIAL:
                case TYPE_DEVICE:
-                       if (stp->st_rdev != MAKEDEV(F_DMAJOR(file), F_DMINOR(file)))
+                       devp = F_RDEV_P(file);
+                       if (stp->st_rdev != MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)))
                                continue;
                        break;
 #ifdef SUPPORT_LINKS
@@ -1155,17 +1233,19 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                return;
        }
 
+#ifdef SUPPORT_HARD_LINKS
        if (preserve_hard_links && F_IS_HLINKED(file)
            && hard_link_check(file, ndx, fname, statret, &st,
                               itemizing, code, HL_CHECK_MASTER))
                return;
+#endif
 
        if (preserve_links && S_ISLNK(file->mode)) {
 #ifdef SUPPORT_LINKS
                const char *sl = F_SYMLINK(file);
                if (safe_symlinks && unsafe_symlink(sl, fname)) {
                        if (verbose) {
-                               if (the_file_list->count == 1)
+                               if (solo_file)
                                        fname = f_name(file, NULL);
                                rprintf(FINFO,
                                        "ignoring unsafe symlink %s -> \"%s\"\n",
@@ -1185,8 +1265,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                if (itemizing)
                                        itemize(file, ndx, 0, &st, 0, 0, NULL);
                                set_file_attrs(fname, file, &st, maybe_ATTRS_REPORT);
+#ifdef SUPPORT_HARD_LINKS
                                if (preserve_hard_links && F_IS_HLINKED(file))
                                        hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                                if (remove_source_files == 1)
                                        goto return_with_success;
                                return;
@@ -1211,10 +1293,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        } else if (j >= 0)
                                statret = 1;
                }
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file)
                    && hard_link_check(file, ndx, fname, -1, &st,
                                       itemizing, code, HL_SKIP))
                        return;
+#endif
                if (do_symlink(sl, fname) != 0) {
                        rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
                                full_fname(fname), sl);
@@ -1226,8 +1310,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        }
                        if (code != FNONE && verbose)
                                rprintf(code, "%s -> %s\n", fname, sl);
+#ifdef SUPPORT_HARD_LINKS
                        if (preserve_hard_links && F_IS_HLINKED(file))
                                hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                        /* This does not check remove_source_files == 1
                         * because this is one of the items that the old
                         * --remove-sent-files option would remove. */
@@ -1240,7 +1326,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
        if ((am_root && preserve_devices && IS_DEVICE(file->mode))
         || (preserve_specials && IS_SPECIAL(file->mode))) {
-               dev_t rdev = MAKEDEV(F_DMAJOR(file), F_DMINOR(file));
+               uint32 *devp = F_RDEV_P(file);
+               dev_t rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
                if (statret == 0) {
                        char *t;
                        if (IS_DEVICE(file->mode)) {
@@ -1253,14 +1340,16 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                t = "special file";
                        }
                        if (statret == 0
-                        && (st.st_mode & ~CHMOD_BITS) == (file->mode & ~CHMOD_BITS)
+                        && (unsigned)(st.st_mode & ~CHMOD_BITS) == (file->mode & ~CHMOD_BITS)
                         && st.st_rdev == rdev) {
                                /* The device or special file is identical. */
                                if (itemizing)
                                        itemize(file, ndx, 0, &st, 0, 0, NULL);
                                set_file_attrs(fname, file, &st, maybe_ATTRS_REPORT);
+#ifdef SUPPORT_HARD_LINKS
                                if (preserve_hard_links && F_IS_HLINKED(file))
                                        hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                                if (remove_source_files == 1)
                                        goto return_with_success;
                                return;
@@ -1283,13 +1372,16 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        } else if (j >= 0)
                                statret = 1;
                }
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file)
                    && hard_link_check(file, ndx, fname, -1, &st,
                                       itemizing, code, HL_SKIP))
                        return;
+#endif
                if (verbose > 2) {
-                       rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
-                               fname, (int)file->mode, (int)rdev);
+                       rprintf(FINFO, "mknod(%s, 0%o, [%ld,%ld])\n",
+                               fname, (int)file->mode,
+                               (long)major(rdev), (long)minor(rdev));
                }
                if (do_mknod(fname, file->mode, rdev) < 0) {
                        rsyserr(FERROR, errno, "mknod %s failed",
@@ -1302,8 +1394,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        }
                        if (code != FNONE && verbose)
                                rprintf(code, "%s\n", fname);
+#ifdef SUPPORT_HARD_LINKS
                        if (preserve_hard_links && F_IS_HLINKED(file))
                                hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                        if (remove_source_files == 1)
                                goto return_with_success;
                }
@@ -1311,7 +1405,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
        }
 
        if (!S_ISREG(file->mode)) {
-               if (the_file_list->count == 1)
+               if (solo_file)
                        fname = f_name(file, NULL);
                rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
                return;
@@ -1319,7 +1413,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 
        if (max_size && F_LENGTH(file) > max_size) {
                if (verbose > 1) {
-                       if (the_file_list->count == 1)
+                       if (solo_file)
                                fname = f_name(file, NULL);
                        rprintf(FINFO, "%s is over max-size\n", fname);
                }
@@ -1327,7 +1421,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
        }
        if (min_size && F_LENGTH(file) < min_size) {
                if (verbose > 1) {
-                       if (the_file_list->count == 1)
+                       if (solo_file)
                                fname = f_name(file, NULL);
                        rprintf(FINFO, "%s is under min-size\n", fname);
                }
@@ -1400,10 +1494,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
        }
 
        if (statret != 0) {
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file)
                    && hard_link_check(file, ndx, fname, statret, &st,
                                       itemizing, code, HL_SKIP))
                        return;
+#endif
                if (stat_errno == ENOENT)
                        goto notify_others;
                rsyserr(FERROR, stat_errno, "recv_generator: failed to stat %s",
@@ -1428,8 +1524,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                                0, 0, NULL);
                }
                set_file_attrs(fname, file, &st, maybe_ATTRS_REPORT);
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file))
                        hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                if (remove_source_files != 1)
                        return;
          return_with_success:
@@ -1463,10 +1561,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
                        full_fname(fnamecmp));
          pretend_missing:
                /* pretend the file didn't exist */
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file)
                    && hard_link_check(file, ndx, fname, statret, &st,
                                       itemizing, code, HL_SKIP))
                        return;
+#endif
                statret = real_ret = -1;
                goto notify_others;
        }
@@ -1523,8 +1623,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
        }
 
        if (!do_xfers) {
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links && F_IS_HLINKED(file))
                        hard_link_cluster(file, ndx, itemizing, code, -1);
+#endif
                return;
        }
        if (read_batch)
@@ -1582,6 +1684,7 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
                maybe_ATTRS_REPORT = ATTRS_REPORT;
                code = FINFO;
        }
+       solo_file = local_name != NULL;
 
        if (verbose > 2) {
                rprintf(FINFO, "generator starting pid=%ld count=%d\n",
@@ -1590,8 +1693,12 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
 
        if (delete_before && !local_name && flist->count > 0)
                do_delete_pass(flist);
-       if (delete_during == 2)
-               start_delete_temp();
+       if (delete_during == 2) {
+               deldelay_size = BIGPATHBUFLEN * 4;
+               deldelay_buf = new_array(char, deldelay_size);
+               if (!deldelay_buf)
+                       out_of_memory("delete-delay");
+       }
        do_progress = 0;
 
        if (append_mode || whole_file < 0)
@@ -1639,8 +1746,10 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
                }
 #endif
 
+#ifdef SUPPORT_HARD_LINKS
                if (preserve_hard_links)
                        check_for_finished_hlinks(itemizing, code);
+#endif
 
                if (allowed_lull && !(i % lull_mod))
                        maybe_send_keepalive();
@@ -1661,7 +1770,7 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
        make_backups = 0; /* avoid a duplicate backup for inplace processing */
 
        if (verbose > 2)
-               rprintf(FINFO,"generate_files phase=%d\n",phase);
+               rprintf(FINFO, "generate_files phase=%d\n", phase);
 
        write_int(f_out, NDX_DONE);
 
@@ -1683,7 +1792,7 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
        make_backups = save_make_backups;
 
        if (verbose > 2)
-               rprintf(FINFO,"generate_files phase=%d\n",phase);
+               rprintf(FINFO, "generate_files phase=%d\n", phase);
 
        write_int(f_out, NDX_DONE);
        /* Reduce round-trip lag-time for a useless delay-updates phase. */
@@ -1704,8 +1813,8 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
        }
 
        do_progress = save_do_progress;
-       if (delete_delay_fp)
-               delayed_deletions(fbuf);
+       if (delete_during == 2)
+               do_delayed_deletions(fbuf);
        if (delete_after && !local_name && flist->count > 0)
                do_delete_pass(flist);
 
@@ -1748,5 +1857,5 @@ void generate_files(int f_out, struct file_list *flist, char *local_name)
        }
 
        if (verbose > 2)
-               rprintf(FINFO,"generate_files finished\n");
+               rprintf(FINFO, "generate_files finished\n");
 }