X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/blobdiff_plain/83f53948103af8dd2312f53173ee6733910f903f..1b42f628f495ff0cdaa8a7c219d8ce33192281fe:/flist.c diff --git a/flist.c b/flist.c index bd26b798..42518d9a 100644 --- a/flist.c +++ b/flist.c @@ -21,6 +21,7 @@ */ #include "rsync.h" +#include "ifuncs.h" #include "rounding.h" #include "io.h" @@ -52,6 +53,7 @@ extern int preserve_devices; extern int preserve_specials; extern int uid_ndx; extern int gid_ndx; +extern int eol_nulls; extern int relative_paths; extern int implied_dirs; extern int file_extra_cnt; @@ -63,6 +65,7 @@ extern int copy_unsafe_links; extern int protocol_version; extern int sanitize_paths; extern struct stats stats; +extern char *filesfrom_host; extern char curr_dir[MAXPATHLEN]; @@ -73,6 +76,7 @@ extern struct filter_list_struct server_filter_list; #ifdef ICONV_OPTION extern int ic_ndx; +extern int filesfrom_convert; extern int need_unsorted_flist; extern iconv_t ic_send, ic_recv; #endif @@ -89,6 +93,18 @@ int flist_cnt = 0; /* how many (non-tmp) file list objects exist */ int file_total = 0; /* total of all active items over all file-lists */ int flist_eof = 0; /* all the file-lists are now known */ +/* Starting from protocol version 26, we always use 64-bit ino_t and dev_t + * internally, even if this platform does not allow files to have 64-bit inums. + * The only exception is if we're on a platform with no 64-bit type at all. + * + * Because we use read_longint() to get these off the wire, if you transfer + * devices or (for protocols < 30) hardlinks with dev or inum > 2**32 to a + * machine with no 64-bit types then you will get an overflow error. + * + * Note that if you transfer devices from a 64-bit-devt machine (say, Solaris) + * to a 32-bit-devt machine (say, Linux-2.2/x86) then the device numbers will + * be truncated. But it's a kind of silly thing to do anyhow. */ + /* The tmp_* vars are used as a cache area by make_file() to store data * that the sender doesn't need to remember in its file list. The data * will survive just long enough to be used by send_file_entry(). */ @@ -101,6 +117,7 @@ static char tmp_sum[MAX_DIGEST_LEN]; static char empty_sum[MAX_DIGEST_LEN]; static int flist_count_offset; /* for --delete --progress */ static int dir_count = 0; +static int high_hlink_ndx; static void clean_flist(struct file_list *flist, int strip_root); static void output_flist(struct file_list *flist); @@ -344,7 +361,7 @@ int push_pathname(const char *dir, int len) return 1; } -static void send_file_entry(int f, struct file_struct *file, int ndx) +static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ndx) { static time_t modtime; static mode_t mode; @@ -362,22 +379,21 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) #ifdef ICONV_OPTION if (ic_send != (iconv_t)-1) { - ICONV_CONST char *ibuf; - char *obuf = fname; - size_t ocnt = MAXPATHLEN, icnt; - - iconv(ic_send, NULL,0, NULL,0); - if ((ibuf = (ICONV_CONST char *)file->dirname) != NULL) { - icnt = strlen(ibuf); - ocnt--; /* pre-subtract the space for the '/' */ - if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) - goto convert_error; - *obuf++ = '/'; - } - - ibuf = (ICONV_CONST char *)file->basename; - icnt = strlen(ibuf); - if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) { + xbuf outbuf, inbuf; + + INIT_CONST_XBUF(outbuf, fname); + + if (file->dirname) { + INIT_XBUF_STRLEN(inbuf, (char*)file->dirname); + outbuf.size -= 2; /* Reserve room for '/' & 1 more char. */ + if (iconvbufs(ic_send, &inbuf, &outbuf, 0) < 0) + goto convert_error; + outbuf.size += 2; + outbuf.buf[outbuf.len++] = '/'; + } + + INIT_XBUF_STRLEN(inbuf, (char*)file->basename); + if (iconvbufs(ic_send, &inbuf, &outbuf, 0) < 0) { convert_error: io_error |= IOERR_GENERAL; rprintf(FINFO, @@ -385,7 +401,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) who_am_i(), f_name(file, fname), strerror(errno)); return; } - *obuf = '\0'; + outbuf.buf[outbuf.len] = '\0'; } else #endif f_name(file, fname); @@ -450,10 +466,11 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) #ifdef SUPPORT_HARD_LINKS if (tmp_dev != 0) { if (protocol_version >= 30) { - struct idev_node *np = idev_node(tmp_dev, tmp_ino); + struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino); first_hlink_ndx = (int32)(long)np->data - 1; if (first_hlink_ndx < 0) { - np->data = (void*)(long)(ndx + 1); + high_hlink_ndx = ndx + first_ndx; + np->data = (void*)(long)(high_hlink_ndx + 1); xflags |= XMIT_HLINK_FIRST; } xflags |= XMIT_HLINKED; @@ -504,8 +521,9 @@ static void send_file_entry(int f, struct file_struct *file, int ndx) write_buf(f, fname + l1, l2); if (first_hlink_ndx >= 0) { - write_varint30(f, first_hlink_ndx); - goto the_end; + write_varint(f, first_hlink_ndx); + if (first_hlink_ndx >= first_ndx) + goto the_end; } write_varlong30(f, F_LENGTH(file), 3); @@ -650,25 +668,19 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef ICONV_OPTION if (ic_recv != (iconv_t)-1) { - char *obuf = thisname; - ICONV_CONST char *ibuf = (ICONV_CONST char *)lastname; - size_t ocnt = MAXPATHLEN, icnt = basename_len; + xbuf outbuf, inbuf; - if (icnt >= MAXPATHLEN) { - errno = E2BIG; - goto convert_error; - } + INIT_CONST_XBUF(outbuf, thisname); + INIT_XBUF(inbuf, lastname, basename_len, -1); - iconv(ic_recv, NULL,0, NULL,0); - if (iconv(ic_recv, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) { - convert_error: + if (iconvbufs(ic_recv, &inbuf, &outbuf, 0) < 0) { io_error |= IOERR_GENERAL; rprintf(FINFO, "[%s] cannot convert filename: %s (%s)\n", who_am_i(), lastname, strerror(errno)); - obuf = thisname; + outbuf.len = 0; } - *obuf = '\0'; + outbuf.buf[outbuf.len] = '\0'; } #endif @@ -693,33 +705,34 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef SUPPORT_HARD_LINKS if (protocol_version >= 30 && BITS_SETnUNSET(xflags, XMIT_HLINKED, XMIT_HLINK_FIRST)) { - struct file_struct *first; - first_hlink_ndx = read_varint30(f); - if (first_hlink_ndx < 0 || first_hlink_ndx >= flist->used) { + first_hlink_ndx = read_varint(f); + if (first_hlink_ndx < 0 || first_hlink_ndx >= flist->ndx_start + flist->used) { rprintf(FERROR, "hard-link reference out of range: %d (%d)\n", - first_hlink_ndx, flist->used); + first_hlink_ndx, flist->ndx_start + flist->used); exit_cleanup(RERR_PROTOCOL); } - first = flist->files[first_hlink_ndx]; - file_length = F_LENGTH(first); - modtime = first->modtime; - mode = first->mode; - if (uid_ndx) - uid = F_OWNER(first); - if (gid_ndx) - gid = F_GROUP(first); - if ((preserve_devices && IS_DEVICE(mode)) - || (preserve_specials && IS_SPECIAL(mode))) { - uint32 *devp = F_RDEV_P(first); - rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); - extra_len += DEV_EXTRA_CNT * EXTRA_LEN; + if (first_hlink_ndx >= flist->ndx_start) { + struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; + file_length = F_LENGTH(first); + modtime = first->modtime; + mode = first->mode; + if (uid_ndx) + uid = F_OWNER(first); + if (gid_ndx) + gid = F_GROUP(first); + if ((preserve_devices && IS_DEVICE(mode)) + || (preserve_specials && IS_SPECIAL(mode))) { + uint32 *devp = F_RDEV_P(first); + rdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); + extra_len += DEV_EXTRA_CNT * EXTRA_LEN; + } + if (preserve_links && S_ISLNK(mode)) + linkname_len = strlen(F_SYMLINK(first)) + 1; + else + linkname_len = 0; + goto create_object; } - if (preserve_links && S_ISLNK(mode)) - linkname_len = strlen(F_SYMLINK(first)) + 1; - else - linkname_len = 0; - goto create_object; } #endif @@ -808,7 +821,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, if (protocol_version < 28 && S_ISREG(mode)) xflags |= XMIT_HLINKED; if (xflags & XMIT_HLINKED) - extra_len += EXTRA_LEN; + extra_len += (inc_recurse+1) * EXTRA_LEN; } #endif @@ -912,8 +925,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef SUPPORT_LINKS if (linkname_len) { bp = (char*)file->basename + basename_len; - if (first_hlink_ndx >= 0) { - struct file_struct *first = flist->files[first_hlink_ndx]; + if (first_hlink_ndx >= flist->ndx_start) { + struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; memcpy(bp, F_SYMLINK(first), linkname_len); } else read_sbuf(f, bp, linkname_len - 1); @@ -925,11 +938,14 @@ static struct file_struct *recv_file_entry(struct file_list *flist, #ifdef SUPPORT_HARD_LINKS if (preserve_hard_links && xflags & XMIT_HLINKED) { if (protocol_version >= 30) { - F_HL_GNUM(file) = xflags & XMIT_HLINK_FIRST - ? flist->used : first_hlink_ndx; + if (xflags & XMIT_HLINK_FIRST) { + high_hlink_ndx = flist->ndx_start + flist->used; + F_HL_GNUM(file) = high_hlink_ndx; + } else + F_HL_GNUM(file) = first_hlink_ndx; } else { static int32 cnt = 0; - struct idev_node *np; + struct ht_int64_node *np; int64 ino; int32 ndx; if (protocol_version < 26) { @@ -940,7 +956,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, dev = read_longint(f); ino = read_longint(f); } - np = idev_node(dev, ino); + np = idev_find(dev, ino); ndx = (int32)(long)np->data - 1; if (ndx < 0) { ndx = cnt++; @@ -958,8 +974,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, /* Prior to 28, we get a useless set of nulls. */ bp = tmp_sum; } - if (first_hlink_ndx >= 0) { - struct file_struct *first = flist->files[first_hlink_ndx]; + if (first_hlink_ndx >= flist->ndx_start) { + struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; memcpy(bp, F_SUM(first), checksum_len); } else read_buf(f, bp, checksum_len); @@ -980,34 +996,24 @@ static struct file_struct *recv_file_entry(struct file_list *flist, return file; } -/** - * Create a file_struct for a named file by reading its stat() - * information and performing extensive checks against global - * options. +/* Create a file_struct for a named file by reading its stat() information + * and performing extensive checks against global options. * - * @return the new file, or NULL if there was an error or this file - * should be excluded. - * - * @todo There is a small optimization opportunity here to avoid - * stat()ing the file in some circumstances, which has a certain cost. - * We are called immediately after doing readdir(), and so we may - * already know the d_type of the file. We could for example avoid - * statting directories if we're not recursing, but this is not a very - * important case. Some systems may not have d_type. - **/ + * Returns a pointer to the new file struct, or NULL if there was an error + * or this file should be excluded. */ struct file_struct *make_file(const char *fname, struct file_list *flist, STRUCT_STAT *stp, int flags, int filter_level) { static char *lastdir; static int lastdir_len = -1; struct file_struct *file; - STRUCT_STAT st; char thisname[MAXPATHLEN]; char linkname[MAXPATHLEN]; int alloc_len, basename_len, linkname_len; int extra_len = file_extra_cnt * EXTRA_LEN; const char *basename; alloc_pool_t *pool; + STRUCT_STAT st; char *bp; if (strlcpy(thisname, fname, sizeof thisname) >= sizeof thisname) { @@ -1285,7 +1291,7 @@ static struct file_struct *send_file_name(int f, struct file_list *flist, flist_expand(flist, 1); flist->files[flist->used++] = file; if (f >= 0) { - send_file_entry(f, file, flist->used - 1); + send_file_entry(f, file, flist->used - 1, flist->ndx_start); #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { send_acl(&sx, f); @@ -1544,6 +1550,7 @@ void send_extra_file_list(int f, int at_least) while (future_cnt < at_least) { struct file_struct *file = dir_flist->sorted[send_dir_ndx]; int dir_ndx, dstart = dir_count; + const char *pathname = F_PATHNAME(file); int32 *dp; flist = flist_new(0, "send_extra_file_list"); @@ -1568,7 +1575,8 @@ void send_extra_file_list(int f, int at_least) && dir_flist->sorted[dir_ndx]->flags & FLAG_DUPLICATE) { send_dir_ndx = dir_ndx; file = dir_flist->sorted[dir_ndx]; - send1extra(f, file, flist); + if (F_PATHNAME(file) != pathname) + send1extra(f, file, flist); dp = F_DIR_NODE_P(file); } @@ -1636,6 +1644,12 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) int64 start_write; int use_ff_fd = 0; int flags, disable_buffering; + int reading_remotely = filesfrom_host != NULL; + int rl_flags = (reading_remotely ? 0 : RL_DUMP_COMMENTS) +#ifdef ICONV_OPTION + | (filesfrom_convert ? RL_CONVERT : 0) +#endif + | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0); rprintf(FLOG, "building file list\n"); if (show_filelist_p()) @@ -1676,7 +1690,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) int is_dot_dir; if (use_ff_fd) { - if (read_filesfrom_line(filesfrom_fd, fbuf) == 0) + if (read_line(filesfrom_fd, fbuf, sizeof fbuf, rl_flags) == 0) break; sanitize_path(fbuf, fbuf, "", 0, NULL); } else { @@ -1782,16 +1796,11 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) dirlen = dir ? strlen(dir) : 0; if (dirlen != lastdir_len || memcmp(lastdir, dir, dirlen) != 0) { if (!push_pathname(dir ? strdup(dir) : NULL, dirlen)) - goto push_error; + continue; lastdir = pathname; lastdir_len = pathname_len; - } else if (!push_pathname(lastdir, lastdir_len)) { - push_error: - io_error |= IOERR_GENERAL; - rsyserr(FERROR, errno, "push_dir %s failed in %s", - full_fname(dir), curr_dir); + } else if (!push_pathname(lastdir, lastdir_len)) continue; - } if (fn != fbuf) memmove(fbuf, fn, len + 1); @@ -1962,13 +1971,13 @@ struct file_list *recv_file_list(int f) start_read = stats.total_read; - flist = flist_new(0, "recv_file_list"); - #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && protocol_version < 30) + if (preserve_hard_links && !first_flist) init_hard_links(); #endif + flist = flist_new(0, "recv_file_list"); + if (inc_recurse) { if (flist->ndx_start == 0) dir_flist = flist_new(FLIST_TEMP, "recv_file_list"); @@ -2174,12 +2183,10 @@ struct file_list *flist_new(int flags, char *msg) { struct file_list *flist; - flist = new(struct file_list); + flist = new0(struct file_list); if (!flist) out_of_memory(msg); - memset(flist, 0, sizeof flist[0]); - if (flags & FLIST_TEMP) { if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0, out_of_memory, POOL_INTERN))) @@ -2198,6 +2205,8 @@ struct file_list *flist_new(int flags, char *msg) flist->file_pool = first_flist->file_pool; flist->ndx_start = first_flist->prev->ndx_end + 2; + if (flist->ndx_start <= high_hlink_ndx) + flist->ndx_start = high_hlink_ndx + 1; flist->prev = first_flist->prev; flist->prev->next = first_flist->prev = flist; @@ -2296,19 +2305,20 @@ static void clean_flist(struct file_list *flist, int strip_root) int keep, drop; /* If one is a dir and the other is not, we want to * keep the dir because it might have contents in the - * list. */ + * list. Otherwise keep the first one. */ if (S_ISDIR(file->mode)) { struct file_struct *fp = flist->sorted[j]; - if (!S_ISDIR(fp->mode)) + if (!S_ISDIR(fp->mode) || !(fp->flags & FLAG_XFER_DIR)) keep = i, drop = j; - else + else { + if (am_sender) + file->flags |= FLAG_DUPLICATE; keep = j, drop = i; + } } else keep = j, drop = i; - if (am_sender) - flist->sorted[drop]->flags |= FLAG_DUPLICATE; - else { + if (!am_sender) { if (verbose > 1) { rprintf(FINFO, "removing duplicate name %s from file list (%d)\n",