From d4d6646aae6d123e1ec013f2354edf843dc06e29 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Mon, 3 Sep 2007 21:21:17 +0000 Subject: [PATCH] Changes to allow hard-linking to work in inc_recurse mode. --- flist.c | 63 +++++++++-------- generator.c | 22 ++++-- hlink.c | 195 +++++++++++++++++++++++++++++++++++++--------------- main.c | 5 +- 4 files changed, 192 insertions(+), 93 deletions(-) diff --git a/flist.c b/flist.c index d694b451..29d3040d 100644 --- a/flist.c +++ b/flist.c @@ -356,7 +356,7 @@ int push_pathname(const char *dir, int len) return 1; } -static void send_file_entry(int f, struct file_struct *file, int ndx) +static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ndx) { static time_t modtime; static mode_t mode; @@ -465,7 +465,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino); first_hlink_ndx = (int32)(long)np->data - 1; if (first_hlink_ndx < 0) { - np->data = (void*)(long)(ndx + 1); + np->data = (void*)(long)(ndx + first_ndx + 1); xflags |= XMIT_HLINK_FIRST; } xflags |= XMIT_HLINKED; @@ -517,7 +517,8 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) if (first_hlink_ndx >= 0) { write_varint(f, first_hlink_ndx); - goto the_end; + if (first_hlink_ndx >= first_ndx) + goto the_end; } write_varlong30(f, F_LENGTH(file), 3); @@ -705,33 +706,34 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef SUPPORT_HARD_LINKS if (protocol_version >= 30 && BITS_SETnUNSET(xflags, XMIT_HLINKED, XMIT_HLINK_FIRST)) { - struct file_struct *first; first_hlink_ndx = read_varint(f); - if (first_hlink_ndx < 0 || first_hlink_ndx >= flist->used) { + if (first_hlink_ndx < 0 || first_hlink_ndx >= flist->ndx_start + flist->used) { rprintf(FERROR, "hard-link reference out of range: %d (%d)\n", - first_hlink_ndx, flist->used); + first_hlink_ndx, flist->ndx_start + flist->used); exit_cleanup(RERR_PROTOCOL); } - first = flist->files[first_hlink_ndx]; - file_length = F_LENGTH(first); - modtime = first->modtime; - mode = first->mode; - if (uid_ndx) - uid = F_OWNER(first); - if (gid_ndx) - gid = F_GROUP(first); - if ((preserve_devices && IS_DEVICE(mode)) - || (preserve_specials && IS_SPECIAL(mode))) { - uint32 *devp = F_RDEV_P(first); - rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); - extra_len += DEV_EXTRA_CNT * EXTRA_LEN; + if (first_hlink_ndx >= flist->ndx_start) { + struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; + file_length = F_LENGTH(first); + modtime = first->modtime; + mode = first->mode; + if (uid_ndx) + uid = F_OWNER(first); + if (gid_ndx) + gid = F_GROUP(first); + if ((preserve_devices && IS_DEVICE(mode)) + || (preserve_specials && IS_SPECIAL(mode))) { + uint32 *devp = F_RDEV_P(first); + rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); + extra_len += DEV_EXTRA_CNT * EXTRA_LEN; + } + if (preserve_links && S_ISLNK(mode)) + linkname_len = strlen(F_SYMLINK(first)) + 1; + else + linkname_len = 0; + goto create_object; } - if (preserve_links && S_ISLNK(mode)) - linkname_len = strlen(F_SYMLINK(first)) + 1; - else - linkname_len = 0; - goto create_object; } #endif @@ -820,7 +822,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, if (protocol_version < 28 && S_ISREG(mode)) xflags |= XMIT_HLINKED; if (xflags & XMIT_HLINKED) - extra_len += EXTRA_LEN; + extra_len += (inc_recurse+1) * EXTRA_LEN; } #endif @@ -924,7 +926,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef SUPPORT_LINKS if (linkname_len) { bp = (char*)file->basename + basename_len; - if (first_hlink_ndx >= 0) { + if (first_hlink_ndx >= flist->ndx_start) { struct file_struct *first = flist->files[first_hlink_ndx]; memcpy(bp, F_SYMLINK(first), linkname_len); } else @@ -938,7 +940,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, if (preserve_hard_links && xflags & XMIT_HLINKED) { if (protocol_version >= 30) { F_HL_GNUM(file) = xflags & XMIT_HLINK_FIRST - ? flist->used : first_hlink_ndx; + ? flist->ndx_start + flist->used + : first_hlink_ndx; } else { static int32 cnt = 0; struct ht_int64_node *np; @@ -970,7 +973,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, /* Prior to 28, we get a useless set of nulls. */ bp = tmp_sum; } - if (first_hlink_ndx >= 0) { + if (first_hlink_ndx >= flist->ndx_start) { struct file_struct *first = flist->files[first_hlink_ndx]; memcpy(bp, F_SUM(first), checksum_len); } else @@ -1287,7 +1290,7 @@ static struct file_struct *send_file_name(int f, struct file_list *flist, flist_expand(flist, 1); flist->files[flist->used++] = file; if (f >= 0) { - send_file_entry(f, file, flist->used - 1); + send_file_entry(f, file, flist->used - 1, flist->ndx_start); #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { send_acl(&sx, f); @@ -1965,7 +1968,7 @@ struct file_list *recv_file_list(int f) start_read = stats.total_read; #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && protocol_version < 30) + if (preserve_hard_links && !first_flist) init_hard_links(); #endif diff --git a/generator.c b/generator.c index e91e3a6f..4963d514 100644 --- a/generator.c +++ b/generator.c @@ -1370,8 +1370,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, statret = 1; } #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_HLINK_NOT_LAST(file)) + if (preserve_hard_links && F_HLINK_NOT_LAST(file)) { + cur_flist->in_progress++; goto cleanup; + } #endif if (do_symlink(sl, fname) != 0) { rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed", @@ -1447,8 +1449,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, statret = 1; } #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_HLINK_NOT_LAST(file)) + if (preserve_hard_links && F_HLINK_NOT_LAST(file)) { + cur_flist->in_progress++; goto cleanup; + } #endif if (verbose > 2) { rprintf(FINFO, "mknod(%s, 0%o, [%ld,%ld])\n", @@ -1567,8 +1571,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, if (statret != 0) { #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_HLINK_NOT_LAST(file)) + if (preserve_hard_links && F_HLINK_NOT_LAST(file)) { + cur_flist->in_progress++; goto cleanup; + } #endif if (stat_errno == ENOENT) goto notify_others; @@ -1630,8 +1636,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, pretend_missing: /* pretend the file didn't exist */ #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_HLINK_NOT_LAST(file)) + if (preserve_hard_links && F_HLINK_NOT_LAST(file)) { + cur_flist->in_progress++; goto cleanup; + } #endif statret = real_ret = -1; goto notify_others; @@ -1793,7 +1801,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) while (preserve_hard_links && (ndx = get_hlink_num()) != -1) { flist = flist_for_ndx(ndx); assert(flist != NULL); - file = flist->files[ndx]; + file = flist->files[ndx - flist->ndx_start]; assert(file->flags & FLAG_HLINKED); finish_hard_link(file, f_name(file, fbuf), NULL, itemizing, code, -1); } @@ -1912,6 +1920,10 @@ void generate_files(int f_out, const char *local_name) dflt_perms = (ACCESSPERMS & ~orig_umask); do { +#ifdef SUPPORT_HARD_LINKS + if (inc_recurse && preserve_hard_links) + match_hard_links(cur_flist); +#endif if (inc_recurse && cur_flist->ndx_start) { struct file_struct *fp = dir_flist->files[cur_flist->parent_ndx]; f_name(fp, fbuf); diff --git a/hlink.c b/hlink.c index a7586938..f82d3b90 100644 --- a/hlink.c +++ b/hlink.c @@ -24,6 +24,8 @@ extern int verbose; extern int dry_run; +extern int am_sender; +extern int inc_recurse; extern int do_xfers; extern int link_dest; extern int preserve_acls; @@ -43,13 +45,21 @@ extern int ic_ndx; /* Starting with protocol 30, we use a simple hashtable on the sending side * for hashing the st_dev and st_ino info. The receiving side gets told * (via flags and a "group index") which items are hard-linked together, so - * we can avoid the pool of dev+inode data. */ + * we can avoid the pool of dev+inode data. For incremental recursion mode, + * the receiver will use a ndx hash to remember old pathnames. */ -struct hashtable *dev_tbl; +static struct hashtable *dev_tbl; + +static struct hashtable *prior_hlinks; + +static struct file_list *hlink_flist; void init_hard_links(void) { - dev_tbl = hashtable_create(16, SIZEOF_INT64 == 8); + if (am_sender || protocol_version < 30) + dev_tbl = hashtable_create(16, SIZEOF_INT64 == 8); + else if (inc_recurse) + prior_hlinks = hashtable_create(1024, 0); } struct ht_int64_node *idev_find(int64 dev, int64 ino) @@ -83,8 +93,8 @@ void idev_destroy(void) static int hlink_compare_gnum(int *int1, int *int2) { - struct file_struct *f1 = cur_flist->sorted[*int1]; - struct file_struct *f2 = cur_flist->sorted[*int2]; + struct file_struct *f1 = hlink_flist->sorted[*int1]; + struct file_struct *f2 = hlink_flist->sorted[*int2]; int32 gnum1 = F_HL_GNUM(f1); int32 gnum2 = F_HL_GNUM(f2); @@ -98,22 +108,39 @@ static void match_gnums(int32 *ndx_list, int ndx_count) { int32 from, prev; struct file_struct *file, *file_next; + struct ht_int32_node *node = NULL; int32 gnum, gnum_next; qsort(ndx_list, ndx_count, sizeof ndx_list[0], (int (*)()) hlink_compare_gnum); for (from = 0; from < ndx_count; from++) { - for (file = cur_flist->sorted[ndx_list[from]], gnum = F_HL_GNUM(file), prev = -1; - from < ndx_count-1; - file = file_next, gnum = gnum_next, from++) /*SHARED ITERATOR*/ - { - file_next = cur_flist->sorted[ndx_list[from+1]]; + file = hlink_flist->sorted[ndx_list[from]]; + gnum = F_HL_GNUM(file); + if (inc_recurse) { + node = hashtable_find(prior_hlinks, gnum, 1); + if (!node->data) { + node->data = new_array0(char, 5); + assert(gnum >= hlink_flist->ndx_start); + } + } + if (gnum >= hlink_flist->ndx_start) { + file->flags |= FLAG_HLINK_FIRST; + prev = -1; + } else if (CVAL(node->data, 0) == 0) { + struct file_list *flist; + prev = IVAL(node->data, 1); + flist = flist_for_ndx(prev); + assert(flist != NULL); + file_next = flist->files[prev - flist->ndx_start]; + file_next->flags &= ~FLAG_HLINK_LAST; + } else + prev = -1; + for ( ; from < ndx_count-1; file = file_next, gnum = gnum_next, from++) { /*SHARED ITERATOR*/ + file_next = hlink_flist->sorted[ndx_list[from+1]]; gnum_next = F_HL_GNUM(file_next); if (gnum != gnum_next) break; - if (prev < 0) - file->flags |= FLAG_HLINK_FIRST; F_HL_PREV(file) = prev; /* The linked list must use raw ndx values. */ #ifdef ICONV_OPTION @@ -121,13 +148,23 @@ static void match_gnums(int32 *ndx_list, int ndx_count) prev = F_NDX(file); else #endif - prev = ndx_list[from]; + prev = ndx_list[from] + hlink_flist->ndx_start; } - if (prev < 0) - file->flags &= ~FLAG_HLINKED; - else { - file->flags |= FLAG_HLINK_LAST; - F_HL_PREV(file) = prev; + if (prev < 0 && !inc_recurse) { + file->flags &= ~(FLAG_HLINKED | FLAG_HLINK_FIRST); + continue; + } + + file->flags |= FLAG_HLINK_LAST; + F_HL_PREV(file) = prev; + if (inc_recurse && CVAL(node->data, 0) == 0) { +#ifdef ICONV_OPTION + if (ic_ndx) + prev = F_NDX(file); + else +#endif + prev = ndx_list[from] + hlink_flist->ndx_start; + SIVAL(node->data, 1, prev); } } } @@ -136,19 +173,21 @@ static void match_gnums(int32 *ndx_list, int ndx_count) * items that have hlink data, sorting them, and matching up identical * values into clusters. These will be a single linked list from last * to first when we're done. */ -void match_hard_links(void) +void match_hard_links(struct file_list *flist) { int i, ndx_count = 0; int32 *ndx_list; - if (!(ndx_list = new_array(int32, cur_flist->used))) + if (!(ndx_list = new_array(int32, flist->used))) out_of_memory("match_hard_links"); - for (i = 0; i < cur_flist->used; i++) { - if (F_IS_HLINKED(cur_flist->sorted[i])) + for (i = 0; i < flist->used; i++) { + if (F_IS_HLINKED(flist->sorted[i])) ndx_list[ndx_count++] = i; } + hlink_flist = flist; + if (ndx_count) match_gnums(ndx_list, ndx_count); @@ -198,6 +237,24 @@ static int maybe_hard_link(struct file_struct *file, int ndx, return -1; } +/* Figure out if a prior entry is still there or if we just have a + * cached name for it. Never called with a FLAG_HLINK_FIRST entry. */ +static char *check_prior(int prev_ndx, int gnum) +{ + struct file_list *flist = flist_for_ndx(prev_ndx); + struct ht_int32_node *node; + + if (flist) { + hlink_flist = flist; + return NULL; + } + + node = hashtable_find(prior_hlinks, gnum, 0); + assert(node != NULL && node->data); + assert(CVAL(node->data, 0) != 0); + return node->data; +} + /* Only called if FLAG_HLINKED is set and FLAG_HLINK_FIRST is not. Returns: * 0 = process the file, 1 = skip the file, -1 = error occurred. */ int hard_link_check(struct file_struct *file, int ndx, const char *fname, @@ -205,42 +262,54 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname, enum logcode code) { STRUCT_STAT prev_st; - char prev_name[MAXPATHLEN], altbuf[MAXPATHLEN], *realname; - int alt_dest, prev_ndx = F_HL_PREV(file); - struct file_struct *prev_file = cur_flist->files[prev_ndx]; - - /* Is the previous link is not complete yet? */ - if (!(prev_file->flags & FLAG_HLINK_DONE)) { - /* Is the previous link being transferred? */ - if (prev_file->flags & FLAG_FILE_SENT) { - /* Add ourselves to the list of files that will be - * updated when the transfer completes, and mark - * ourself as waiting for the transfer. */ - F_HL_PREV(file) = F_HL_PREV(prev_file); - F_HL_PREV(prev_file) = ndx; - file->flags |= FLAG_FILE_SENT; - return 1; + char namebuf[MAXPATHLEN], altbuf[MAXPATHLEN]; + char *realname, *prev_name; + int gnum = F_HL_GNUM(file); + int prev_ndx = F_HL_PREV(file); + + prev_name = realname = check_prior(prev_ndx, gnum); + + if (!prev_name) { + struct file_struct *prev_file = hlink_flist->files[prev_ndx - hlink_flist->ndx_start]; + + /* Is the previous link is not complete yet? */ + if (!(prev_file->flags & FLAG_HLINK_DONE)) { + /* Is the previous link being transferred? */ + if (prev_file->flags & FLAG_FILE_SENT) { + /* Add ourselves to the list of files that will be + * updated when the transfer completes, and mark + * ourself as waiting for the transfer. */ + F_HL_PREV(file) = F_HL_PREV(prev_file); + F_HL_PREV(prev_file) = ndx; + file->flags |= FLAG_FILE_SENT; + cur_flist->in_progress++; + return 1; + } + return 0; } - return 0; - } - /* There is a finished file to link with! */ - if (!(prev_file->flags & FLAG_HLINK_FIRST)) { - /* The previous previous will be marked with FIRST. */ - prev_ndx = F_HL_PREV(prev_file); - prev_file = cur_flist->files[prev_ndx]; - /* Update our previous pointer to point to the first. */ - F_HL_PREV(file) = prev_ndx; + /* There is a finished file to link with! */ + if (!(prev_file->flags & FLAG_HLINK_FIRST)) { + /* The previous previous will be marked with FIRST. */ + prev_ndx = F_HL_PREV(prev_file); + prev_name = realname = check_prior(prev_ndx, gnum); + /* Update our previous pointer to point to the first. */ + F_HL_PREV(file) = prev_ndx; + } } - alt_dest = F_HL_PREV(prev_file); /* alternate value when DONE && FIRST */ - if (alt_dest >= 0 && dry_run) { - pathjoin(prev_name, MAXPATHLEN, basis_dir[alt_dest], - f_name(prev_file, NULL)); - f_name(prev_file, altbuf); - realname = altbuf; - } else { - f_name(prev_file, prev_name); - realname = prev_name; + if (!prev_name) { + struct file_struct *prev_file = hlink_flist->files[prev_ndx - hlink_flist->ndx_start]; + int alt_dest = F_HL_PREV(prev_file); /* alternate value when DONE && FIRST */ + + if (alt_dest >= 0 && dry_run) { + pathjoin(namebuf, MAXPATHLEN, basis_dir[alt_dest], + f_name(prev_file, NULL)); + prev_name = namebuf; + realname = f_name(prev_file, altbuf); + } else { + prev_name = f_name(prev_file, namebuf); + realname = prev_name; + } } if (link_stat(prev_name, &prev_st, 0) < 0) { @@ -338,6 +407,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, STRUCT_STAT st; char alt_name[MAXPATHLEN], *prev_name; const char *our_name; + struct file_list *flist; int prev_statret, ndx, prev_ndx = F_HL_PREV(file); if (stp == NULL && prev_ndx >= 0) { @@ -365,13 +435,16 @@ void finish_hard_link(struct file_struct *file, const char *fname, while ((ndx = prev_ndx) >= 0) { int val; - file = cur_flist->files[ndx]; + flist = flist_for_ndx(ndx); + assert(flist != NULL); + file = flist->files[ndx - flist->ndx_start]; file->flags = (file->flags & ~FLAG_HLINK_FIRST) | FLAG_HLINK_DONE; prev_ndx = F_HL_PREV(file); prev_name = f_name(file, NULL); prev_statret = link_stat(prev_name, &prev_sx.st, 0); val = maybe_hard_link(file, ndx, prev_name, prev_statret, &prev_sx, our_name, stp, fname, itemizing, code); + flist->in_progress--; #ifdef SUPPORT_ACLS if (preserve_acls) free_acl(&prev_sx); @@ -381,5 +454,15 @@ void finish_hard_link(struct file_struct *file, const char *fname, if (remove_source_files == 1 && do_xfers) send_msg_int(MSG_SUCCESS, ndx); } + + if (inc_recurse) { + int gnum = F_HL_GNUM(file); + struct ht_int32_node *node = hashtable_find(prior_hlinks, gnum, 0); + assert(node != NULL && node->data != NULL); + assert(CVAL(node->data, 0) == 0); + free(node->data); + if (!(node->data = strdup(our_name))) + out_of_memory("finish_hard_link"); + } } #endif diff --git a/main.c b/main.c index 0e765fb0..e9675f96 100644 --- a/main.c +++ b/main.c @@ -70,6 +70,7 @@ extern char *shell_cmd; extern char *batch_name; extern char *password_file; extern char curr_dir[MAXPATHLEN]; +extern struct file_list *cur_flist; extern struct filter_list_struct server_filter_list; int local_server = 0; @@ -695,8 +696,8 @@ static int do_recv(int f_in, int f_out, char *local_name) copy_links = copy_dirlinks = 0; #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links) - match_hard_links(); + if (preserve_hard_links && !inc_recurse) + match_hard_links(cur_flist); #endif if (fd_pair(error_pipe) < 0) { -- 2.34.1