all attributes of a file by using a special extended-attribute idiom.
There is also an analogous "fake super" option for an rsync daemon.
+ - Added the --iconv option, which allows rsync to convert filenames from
+ one character set to another during the transfer. (Currently must be
+ manually enabled via configure's --enable-iconv option.)
+
- You may specify --max-delete=0 to a 3.0.0 client as long as the receiving
side is at least version 3.0.0. This means that you can pull from an
older rsync with this option, but pushing to an older rsync will generate
if (!ret || err_msg) {
if (err_msg)
- rwrite(FERROR, err_msg, strlen(err_msg));
+ rwrite(FERROR, err_msg, strlen(err_msg), 0);
else
option_error();
msleep(400);
AC_FUNC_GETPGRP
fi
+AC_ARG_ENABLE(iconv,
+ AC_HELP_STRING([--disable-iconv],
+ [disable rsync's --iconv option]),
+ [], [enable_iconv=$ac_cv_func_iconv_open])
+AH_TEMPLATE([ICONV_OPTION],
+[Define if you want the --iconv option. Specifing a value will set the
+default iconv setting (a NULL means no --iconv processing by default).])
+if test x"$enable_iconv" != x"no"; then
+ if test x"$enable_iconv" = x"yes"; then
+ AC_DEFINE(ICONV_OPTION, NULL)
+ else
+ AC_DEFINE_UNQUOTED(ICONV_OPTION, "$enable_iconv")
+ fi
+ AC_DEFINE(UTF8_CHARSET, "UTF-8", [String to pass to iconv() for the UTF-8 charset.])
+fi
+
AC_CACHE_CHECK([whether chown() modifies symlinks],rsync_cv_chown_modifies_symlink,[
AC_TRY_RUN([
#if HAVE_UNISTD_H
extern struct filter_list_struct filter_list;
extern struct filter_list_struct server_filter_list;
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+extern int need_unsorted_flist;
+extern iconv_t ic_send, ic_recv;
+#endif
+
int io_error;
int checksum_len;
dev_t filesystem_dev; /* used to implement -x */
int l1, l2;
int flags;
- f_name(file, fname);
+#ifdef ICONV_OPTION
+ if (ic_send != (iconv_t)-1) {
+ ICONV_CONST char *ibuf;
+ char *obuf = fname;
+ size_t ocnt = MAXPATHLEN, icnt;
+
+ iconv(ic_send, NULL,0, NULL,0);
+ if ((ibuf = (ICONV_CONST char *)file->dirname) != NULL) {
+ icnt = strlen(ibuf);
+ ocnt--; /* pre-subtract the space for the '/' */
+ if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1)
+ goto convert_error;
+ *obuf++ = '/';
+ }
+
+ ibuf = (ICONV_CONST char *)file->basename;
+ icnt = strlen(ibuf);
+ if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) {
+ convert_error:
+ io_error |= IOERR_GENERAL;
+ rprintf(FINFO,
+ "[%s] cannot convert filename: %s (%s)\n",
+ who_am_i(), f_name(file, fname), strerror(errno));
+ return;
+ }
+ *obuf = '\0';
+ } else
+#endif
+ f_name(file, fname);
flags = file->flags & FLAG_TOP_DIR; /* FLAG_TOP_DIR == XMIT_TOP_DIR */
read_sbuf(f, &thisname[l1], l2);
thisname[l1 + l2] = 0;
- strlcpy(lastname, thisname, MAXPATHLEN);
+ /* Abuse basename_len for a moment... */
+ basename_len = strlcpy(lastname, thisname, MAXPATHLEN);
+
+#ifdef ICONV_OPTION
+ if (ic_recv != (iconv_t)-1) {
+ char *obuf = thisname, *ibuf = lastname;
+ size_t ocnt = MAXPATHLEN, icnt = basename_len;
+
+ if (icnt >= MAXPATHLEN) {
+ errno = E2BIG;
+ goto convert_error;
+ }
+
+ iconv(ic_recv, NULL,0, NULL,0);
+ if (iconv(ic_recv, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) {
+ convert_error:
+ io_error |= IOERR_GENERAL;
+ rprintf(FINFO,
+ "[%s] cannot convert filename: %s (%s)\n",
+ who_am_i(), lastname, strerror(errno));
+ obuf = thisname;
+ }
+ *obuf = '\0';
+ }
+#endif
clean_fname(thisname, 0);
F_OWNER(file) = uid;
if (preserve_gid)
F_GROUP(file) = gid;
+#ifdef ICONV_OPTION
+ if (ic_ndx)
+ F_NDX(file) = flist->count + flist->ndx_start;
+#endif
if (basename != thisname) {
file->dirname = lastdir;
FLAG_DIVERT_DIRS | FLAG_XFER_DIR);
write_byte(f, 0);
- clean_flist(flist, 0, 0);
+#ifdef ICONV_OPTION
+ if (!need_unsorted_flist)
+#endif
+ {
+ flist->sorted = flist->files;
+ clean_flist(flist, 0, 0);
+ }
file_total += flist->count;
future_cnt += flist->count;
stats.flist_size += stats.total_written - start_write;
stats.flist_xfertime = (int64)(end_tv.tv_sec - start_tv.tv_sec) * 1000
+ (end_tv.tv_usec - start_tv.tv_usec) / 1000;
+ /* When converting names, both sides keep an unsorted file-list array
+ * because the names will differ on the sending and receiving sides
+ * (both sides will use the unsorted index number for each item). */
+
/* Sort the list without removing any duplicates in non-incremental
* mode. This allows the receiving side to ask for whatever name it
* kept. For incremental mode, the sender also removes duplicates
* in this initial file-list so that it avoids re-sending duplicated
* directories. */
- clean_flist(flist, 0, inc_recurse);
+#ifdef ICONV_OPTION
+ if (need_unsorted_flist) {
+ if (inc_recurse) {
+ if (!(flist->sorted = new_array(struct file_struct *, flist->count)))
+ out_of_memory("recv_file_list");
+ memcpy(flist->sorted, flist->files,
+ flist->count * sizeof (struct file_struct*));
+ clean_flist(flist, 0, 1);
+ }
+ } else
+#endif
+ {
+ flist->sorted = flist->files;
+ clean_flist(flist, 0, inc_recurse);
+ }
file_total += flist->count;
if (!numeric_ids && !inc_recurse)
if (show_filelist_p())
finish_filelist_progress(flist);
+#ifdef ICONV_OPTION
+ if (need_unsorted_flist) {
+ /* Create an extra array of index pointers that we can sort for
+ * the generator's use (for wading through the files in sorted
+ * order and for calling flist_find()). We keep the "files"
+ * list unsorted for our exchange of index numbers with the
+ * other side (since their names may not sort the same). */
+ if (!(flist->sorted = new_array(struct file_struct *, flist->count)))
+ out_of_memory("recv_file_list");
+ memcpy(flist->sorted, flist->files,
+ flist->count * sizeof (struct file_struct*));
+ } else
+#endif
+ flist->sorted = flist->files;
+
clean_flist(flist, relative_paths, 1);
if (inc_recurse) {
- qsort(dir_flist->files + dstart, dir_flist->count - dstart,
- sizeof dir_flist->files[0], (int (*)())file_compare);
+ dir_flist->sorted = dir_flist->files;
+ qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
+ sizeof dir_flist->sorted[0], (int (*)())file_compare);
} else if (f >= 0)
recv_uid_list(f, flist);
while (low <= high) {
mid = (low + high) / 2;
- if (F_IS_ACTIVE(flist->files[mid]))
+ if (F_IS_ACTIVE(flist->sorted[mid]))
mid_up = mid;
else {
/* Scan for the next non-empty entry using the cached
* distance values. If the value isn't fully up-to-
* date, update it. */
- mid_up = mid + F_DEPTH(flist->files[mid]);
- if (!F_IS_ACTIVE(flist->files[mid_up])) {
+ mid_up = mid + F_DEPTH(flist->sorted[mid]);
+ if (!F_IS_ACTIVE(flist->sorted[mid_up])) {
do {
- mid_up += F_DEPTH(flist->files[mid_up]);
- } while (!F_IS_ACTIVE(flist->files[mid_up]));
- F_DEPTH(flist->files[mid]) = mid_up - mid;
+ mid_up += F_DEPTH(flist->sorted[mid_up]);
+ } while (!F_IS_ACTIVE(flist->sorted[mid_up]));
+ F_DEPTH(flist->sorted[mid]) = mid_up - mid;
}
if (mid_up > high) {
/* If there's nothing left above us, set high to
* a non-empty entry below us and continue. */
- high = mid - (int)flist->files[mid]->len32;
- if (!F_IS_ACTIVE(flist->files[high])) {
+ high = mid - (int)flist->sorted[mid]->len32;
+ if (!F_IS_ACTIVE(flist->sorted[high])) {
do {
- high -= (int)flist->files[high]->len32;
- } while (!F_IS_ACTIVE(flist->files[high]));
- flist->files[mid]->len32 = mid - high;
+ high -= (int)flist->sorted[high]->len32;
+ } while (!F_IS_ACTIVE(flist->sorted[high]));
+ flist->sorted[mid]->len32 = mid - high;
}
continue;
}
}
- diff = f_name_cmp(flist->files[mid_up], f);
+ diff = f_name_cmp(flist->sorted[mid_up], f);
if (diff == 0) {
if (protocol_version < 29
- && S_ISDIR(flist->files[mid_up]->mode)
+ && S_ISDIR(flist->sorted[mid_up]->mode)
!= S_ISDIR(f->mode))
return -1;
return mid_up;
}
pool_destroy(flist->file_pool);
+ if (flist->sorted && flist->sorted != flist->files)
+ free(flist->sorted);
free(flist->files);
free(flist);
}
return;
}
- qsort(flist->files, flist->count,
- sizeof flist->files[0], (int (*)())file_compare);
+ qsort(flist->sorted, flist->count,
+ sizeof flist->sorted[0], (int (*)())file_compare);
for (i = no_dups? 0 : flist->count; i < flist->count; i++) {
- if (F_IS_ACTIVE(flist->files[i])) {
+ if (F_IS_ACTIVE(flist->sorted[i])) {
prev_i = i;
break;
}
flist->low = prev_i;
while (++i < flist->count) {
int j;
- struct file_struct *file = flist->files[i];
+ struct file_struct *file = flist->sorted[i];
if (!F_IS_ACTIVE(file))
continue;
- if (f_name_cmp(file, flist->files[prev_i]) == 0)
+ if (f_name_cmp(file, flist->sorted[prev_i]) == 0)
j = prev_i;
else if (protocol_version >= 29 && S_ISDIR(file->mode)) {
int save_mode = file->mode;
} else
j = -1;
if (j >= 0) {
- struct file_struct *fp = flist->files[j];
+ struct file_struct *fp = flist->sorted[j];
int keep, drop;
/* If one is a dir and the other is not, we want to
* keep the dir because it might have contents in the
}
/* Make sure we don't lose track of a user-specified
* top directory. */
- flist->files[keep]->flags |= flist->files[drop]->flags
+ flist->sorted[keep]->flags |= flist->sorted[drop]->flags
& (FLAG_TOP_DIR|FLAG_XFER_DIR);
- clear_file(flist->files[drop]);
+ clear_file(flist->sorted[drop]);
if (keep == i) {
if (flist->low == drop) {
for (j = drop + 1;
- j < i && !F_IS_ACTIVE(flist->files[j]);
+ j < i && !F_IS_ACTIVE(flist->sorted[j]);
j++) {}
flist->low = j;
}
/* We need to strip off the leading slashes for relative
* paths, but this must be done _after_ the sorting phase. */
for (i = flist->low; i <= flist->high; i++) {
- struct file_struct *file = flist->files[i];
+ struct file_struct *file = flist->sorted[i];
if (!file->dirname)
continue;
prev_i = 0; /* It's OK that this isn't really true. */
for (i = flist->low; i <= flist->high; i++) {
- struct file_struct *fp, *file = flist->files[i];
+ struct file_struct *fp, *file = flist->sorted[i];
/* This temporarily abuses the F_DEPTH() value for a
* directory that is in a chain that might get pruned.
if (S_ISDIR(file->mode) && F_DEPTH(file)) {
/* Dump empty dirs when coming back down. */
for (j = prev_depth; j >= F_DEPTH(file); j--) {
- fp = flist->files[prev_i];
+ fp = flist->sorted[prev_i];
if (F_DEPTH(fp) >= 0)
break;
prev_i = -F_DEPTH(fp)-1;
ALL_FILTERS)) {
/* Keep dirs through this dir. */
for (j = prev_depth-1; ; j--) {
- fp = flist->files[prev_i];
+ fp = flist->sorted[prev_i];
if (F_DEPTH(fp) >= 0)
break;
prev_i = -F_DEPTH(fp)-1;
} else {
/* Keep dirs through this non-dir. */
for (j = prev_depth; ; j--) {
- fp = flist->files[prev_i];
+ fp = flist->sorted[prev_i];
if (F_DEPTH(fp) >= 0)
break;
prev_i = -F_DEPTH(fp)-1;
}
/* Dump all remaining empty dirs. */
while (1) {
- struct file_struct *fp = flist->files[prev_i];
+ struct file_struct *fp = flist->sorted[prev_i];
if (F_DEPTH(fp) >= 0)
break;
prev_i = -F_DEPTH(fp)-1;
}
for (i = flist->low; i <= flist->high; i++) {
- if (F_IS_ACTIVE(flist->files[i]))
+ if (F_IS_ACTIVE(flist->sorted[i]))
break;
}
flist->low = i;
for (i = flist->high; i >= flist->low; i--) {
- if (F_IS_ACTIVE(flist->files[i]))
+ if (F_IS_ACTIVE(flist->sorted[i]))
break;
}
flist->high = i;
if (do_progress)
flist_count_offset += dirlist->count;
+ dirlist->sorted = dirlist->files;
clean_flist(dirlist, 0, 0);
if (verbose > 3)
extern int backup_suffix_len;
extern struct file_list *cur_flist, *first_flist, *dir_flist;
extern struct filter_list_struct server_filter_list;
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+#endif
int ignore_perishable = 0;
int non_perishable_cnt = 0;
* MAXPATHLEN buffer with the name of the directory in it (the functions we
* call will append names onto the end, but the old dir value will be restored
* on exit). */
-static void delete_in_dir(struct file_list *flist, char *fbuf,
- struct file_struct *file, dev_t *fs_dev)
+static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
{
static int already_warned = 0;
struct file_list *dirlist;
char delbuf[MAXPATHLEN];
int dlen, i;
- if (!flist) {
+ if (!fbuf) {
change_local_filter_dir(NULL, 0, 0);
return;
}
f_name(fp, NULL));
continue;
}
- if (flist_find(flist, fp) < 0) {
+ if (flist_find(cur_flist, fp) < 0) {
f_name(fp, delbuf);
if (delete_during == 2) {
if (!remember_delete(fp, delbuf))
/* This deletes any files on the receiving side that are not present on the
* sending side. This is used by --delete-before and --delete-after. */
-static void do_delete_pass(struct file_list *flist)
+static void do_delete_pass(void)
{
char fbuf[MAXPATHLEN];
STRUCT_STAT st;
if (dry_run > 1 || list_only)
return;
- for (j = 0; j < flist->count; j++) {
- struct file_struct *file = flist->files[j];
+ for (j = 0; j < cur_flist->count; j++) {
+ struct file_struct *file = cur_flist->sorted[j];
if (!(file->flags & FLAG_XFER_DIR))
continue;
|| !S_ISDIR(st.st_mode))
continue;
- delete_in_dir(flist, fbuf, file, &st.st_dev);
+ delete_in_dir(fbuf, file, &st.st_dev);
}
- delete_in_dir(NULL, NULL, NULL, &dev_zero);
+ delete_in_dir(NULL, NULL, &dev_zero);
if (do_progress && !am_server)
rprintf(FINFO, " \r");
}
else if (delete_during && f_out != -1 && !phase && dry_run < 2
&& (file->flags & FLAG_XFER_DIR))
- delete_in_dir(cur_flist, fname, file, &real_sx.st.st_dev);
+ delete_in_dir(fname, file, &real_sx.st.st_dev);
goto cleanup;
}
/* Fix any directory permissions that were modified during the
* transfer and/or re-set any tweaked modified-time values. */
for (i = start, j = 0; i <= end; i++) {
- file = flist->files[i];
+ file = flist->sorted[i];
if (!F_IS_ACTIVE(file) || !S_ISDIR(file->mode)
|| file->flags & FLAG_MISSING_DIR)
continue;
void generate_files(int f_out, const char *local_name)
{
- int i;
+ int i, ndx;
char fbuf[MAXPATHLEN];
int itemizing;
enum logcode code;
rprintf(FINFO, "generator starting pid=%ld\n", (long)getpid());
if (delete_before && !solo_file && cur_flist->count > 0)
- do_delete_pass(cur_flist);
+ do_delete_pass();
if (delete_during == 2) {
deldelay_size = BIGPATHBUFLEN * 4;
deldelay_buf = new_array(char, deldelay_size);
dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
} else
dirdev = MAKEDEV(0, 0);
- delete_in_dir(cur_flist, f_name(fp, fbuf), fp, &dirdev);
+ delete_in_dir(f_name(fp, fbuf), fp, &dirdev);
}
}
for (i = cur_flist->low; i <= cur_flist->high; i++) {
- struct file_struct *file = cur_flist->files[i];
+ struct file_struct *file = cur_flist->sorted[i];
if (!F_IS_ACTIVE(file))
continue;
+#ifdef ICONV_OPTION
+ if (ic_ndx)
+ ndx = F_NDX(file);
+ else
+#endif
+ ndx = i + cur_flist->ndx_start;
+
if (solo_file)
strlcpy(fbuf, solo_file, sizeof fbuf);
else
f_name(file, fbuf);
- recv_generator(fbuf, file, i + cur_flist->ndx_start,
- itemizing, code, f_out);
+ recv_generator(fbuf, file, ndx, itemizing, code, f_out);
/* We need to ensure that any dirs we create have
* writeable permissions during the time we are putting
} while ((cur_flist = cur_flist->next) != NULL);
if (delete_during)
- delete_in_dir(NULL, NULL, NULL, &dev_zero);
+ delete_in_dir(NULL, NULL, &dev_zero);
phase++;
if (verbose > 2)
rprintf(FINFO, "generate_files phase=%d\n", phase);
if (delete_during == 2)
do_delayed_deletions(fbuf);
if (delete_after && !solo_file && file_total > 0)
- do_delete_pass(cur_flist);
+ do_delete_pass();
if ((need_retouch_dir_perms || need_retouch_dir_times)
&& dir_tweaking && (!inc_recurse || delete_during == 2))
extern int maybe_ATTRS_REPORT;
extern char *basis_dir[];
extern struct file_list *cur_flist;
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+#endif
#ifdef SUPPORT_HARD_LINKS
#define HASH_LOAD_LIMIT(size) ((size)*3/4)
-#define FPTR(i) (cur_flist->files[i])
struct ihash_table {
int32 size;
static int hlink_compare_gnum(int *int1, int *int2)
{
- struct file_struct *f1 = FPTR(*int1);
- struct file_struct *f2 = FPTR(*int2);
+ struct file_struct *f1 = cur_flist->sorted[*int1];
+ struct file_struct *f2 = cur_flist->sorted[*int2];
int32 gnum1 = F_HL_GNUM(f1);
int32 gnum2 = F_HL_GNUM(f2);
(int (*)()) hlink_compare_gnum);
for (from = 0; from < ndx_count; from++) {
- for (file = FPTR(ndx_list[from]), gnum = F_HL_GNUM(file), prev = -1;
+ for (file = cur_flist->sorted[ndx_list[from]], gnum = F_HL_GNUM(file), prev = -1;
from < ndx_count-1;
- file = file_next, gnum = gnum_next, prev = ndx_list[from++])
+ file = file_next, gnum = gnum_next, from++)
{
- file_next = FPTR(ndx_list[from+1]);
+ file_next = cur_flist->sorted[ndx_list[from+1]];
gnum_next = F_HL_GNUM(file_next);
if (gnum != gnum_next)
break;
if (prev < 0)
file->flags |= FLAG_HLINK_FIRST;
F_HL_PREV(file) = prev;
+ /* The linked list must use raw ndx values. */
+#ifdef ICONV_OPTION
+ if (ic_ndx)
+ prev = F_NDX(file);
+ else
+#endif
+ prev = ndx_list[from];
}
if (prev < 0)
file->flags &= ~FLAG_HLINKED;
out_of_memory("match_hard_links");
for (i = 0; i < cur_flist->count; i++) {
- if (F_IS_HLINKED(FPTR(i)))
+ if (F_IS_HLINKED(cur_flist->sorted[i]))
ndx_list[ndx_count++] = i;
}
STRUCT_STAT prev_st;
char prev_name[MAXPATHLEN], altbuf[MAXPATHLEN], *realname;
int alt_dest, prev_ndx = F_HL_PREV(file);
- struct file_struct *prev_file = FPTR(prev_ndx);
+ struct file_struct *prev_file = cur_flist->files[prev_ndx];
/* Is the previous link is not complete yet? */
if (!(prev_file->flags & FLAG_HLINK_DONE)) {
if (!(prev_file->flags & FLAG_HLINK_FIRST)) {
/* The previous previous will be marked with FIRST. */
prev_ndx = F_HL_PREV(prev_file);
- prev_file = FPTR(prev_ndx);
+ prev_file = cur_flist->files[prev_ndx];
/* Update our previous pointer to point to the first. */
F_HL_PREV(file) = prev_ndx;
}
while ((ndx = prev_ndx) >= 0) {
int val;
- file = FPTR(ndx);
+ file = cur_flist->files[ndx];
file->flags = (file->flags & ~FLAG_HLINK_FIRST) | FLAG_HLINK_DONE;
prev_ndx = F_HL_PREV(file);
prev_name = f_name(file, NULL);
extern char *filesfrom_host;
extern struct stats stats;
extern struct file_list *cur_flist, *first_flist;
+#ifdef ICONV_OPTION
+extern iconv_t ic_send, ic_recv;
+#endif
const char phase_unknown[] = "unknown";
int ignore_timeout = 0;
static int io_filesfrom_f_in = -1;
static int io_filesfrom_f_out = -1;
static char io_filesfrom_buf[2048];
+#ifdef ICONV_OPTION
+static char iconv_buf[sizeof io_filesfrom_buf / 2];
+#endif
static char *io_filesfrom_bp;
static char io_filesfrom_lastchar;
static int io_filesfrom_buflen;
static void writefd_unbuffered(int fd, const char *buf, size_t len);
static void decrement_active_files(int ndx);
static void decrement_flist_in_progress(int ndx, int redo);
+static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len, int convert);
struct flist_ndx_item {
struct flist_ndx_item *next;
struct msg_list_item {
struct msg_list_item *next;
- int len;
+ char convert;
char buf[1];
};
}
/* Add a message to the pending MSG_* list. */
-static void msg_list_add(struct msg_list *lst, int code, const char *buf, int len)
+static void msg_list_add(struct msg_list *lst, int code, const char *buf, int len, int convert)
{
struct msg_list_item *m;
int sz = len + 4 + sizeof m[0] - 1;
if (!(m = (struct msg_list_item *)new_array(char, sz)))
out_of_memory("msg_list_add");
m->next = NULL;
- m->len = len + 4;
+ m->convert = convert;
SIVAL(m->buf, 0, ((code+MPLEX_BASE)<<24) | len);
memcpy(m->buf + 4, buf, len);
if (lst->tail)
if (am_generator) {
while (msg_queue.head && io_multiplexing_out) {
struct msg_list_item *m = msg_queue.head;
+ int len = IVAL(m->buf, 0) & 0xFFFFFF;
+ int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
if (!(msg_queue.head = m->next))
msg_queue.tail = NULL;
- stats.total_written += m->len;
+ stats.total_written += len + 4;
defer_forwarding_messages++;
- writefd_unbuffered(sock_f_out, m->buf, m->len);
+ mplex_write(sock_f_out, tag, m->buf + 4, len, m->convert);
defer_forwarding_messages--;
free(m);
}
} else {
while (msg_queue.head) {
struct msg_list_item *m = msg_queue.head;
+ int len = IVAL(m->buf, 0) & 0xFFFFFF;
+ int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
if (!(msg_queue.head = m->next))
msg_queue.tail = NULL;
defer_forwarding_messages++;
- writefd_unbuffered(msg_fd_out, m->buf, m->len);
+ mplex_write(msg_fd_out, tag, m->buf + 4, len, m->convert);
defer_forwarding_messages--;
free(m);
}
if (len >= (int)sizeof buf || !am_generator)
goto invalid_msg;
readfd(fd, buf, len);
- send_msg(MSG_DELETED, buf, len);
+ send_msg(MSG_DELETED, buf, len, 1);
break;
case MSG_SUCCESS:
if (len != 4 || !am_generator)
readfd(fd, buf, len);
if (remove_source_files) {
decrement_active_files(IVAL(buf,0));
- send_msg(MSG_SUCCESS, buf, len);
+ send_msg(MSG_SUCCESS, buf, len, 0);
}
if (preserve_hard_links)
flist_ndx_push(&hlink_list, IVAL(buf,0));
if (n >= sizeof buf)
n = sizeof buf - 1;
readfd(fd, buf, n);
- rwrite((enum logcode)tag, buf, n);
+ rwrite((enum logcode)tag, buf, n, !am_generator);
len -= n;
}
break;
}
/* Write an message to a multiplexed stream. If this fails, rsync exits. */
-static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len)
+static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len, int convert)
{
- char buffer[1024];
+ char buffer[BIGPATHBUFLEN]; /* Oversized for use by iconv code. */
size_t n = len;
SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len);
- if (n > sizeof buffer - 4)
+#ifdef ICONV_OPTION
+ if (convert && ic_send == (iconv_t)-1)
+#endif
+ convert = 0;
+
+ if (convert || n > 1024 - 4) /* BIGPATHBUFLEN can handle 1024 bytes */
n = 0;
else
memcpy(buffer + 4, buf, n);
len -= n;
buf += n;
+#ifdef ICONV_OPTION
+ if (convert) {
+ iconv(ic_send, NULL, 0, NULL, 0);
+ defer_forwarding_messages++;
+ while (len) {
+ ICONV_CONST char *ibuf = (ICONV_CONST char *)buf;
+ char *obuf = buffer;
+ size_t ocnt = sizeof buffer;
+ while (len && iconv(ic_send, &ibuf,&len,
+ &obuf,&ocnt) == (size_t)-1) {
+ if (errno == E2BIG || !ocnt)
+ break;
+ *obuf++ = *ibuf++;
+ ocnt--, len--;
+ }
+ n = obuf - buffer;
+ writefd_unbuffered(fd, buffer, n);
+ }
+ if (!--defer_forwarding_messages)
+ msg_flush();
+ } else
+#endif
if (len) {
defer_forwarding_messages++;
writefd_unbuffered(fd, buf, len);
}
}
-int send_msg(enum msgcode code, const char *buf, int len)
+int send_msg(enum msgcode code, const char *buf, int len, int convert)
{
if (msg_fd_out < 0) {
if (!defer_forwarding_messages)
- return io_multiplex_write(code, buf, len);
+ return io_multiplex_write(code, buf, len, convert);
if (!io_multiplexing_out)
return 0;
- msg_list_add(&msg_queue, code, buf, len);
+ msg_list_add(&msg_queue, code, buf, len, convert);
return 1;
}
if (flist_forward_from >= 0)
- msg_list_add(&msg_queue, code, buf, len);
+ msg_list_add(&msg_queue, code, buf, len, convert);
else
- mplex_write(msg_fd_out, code, buf, len);
+ mplex_write(msg_fd_out, code, buf, len, convert);
return 1;
}
{
char numbuf[4];
SIVAL(numbuf, 0, num);
- send_msg(code, numbuf, 4);
+ send_msg(code, numbuf, 4, 0);
}
void wait_for_receiver(void)
return cnt;
}
-/**
- * Read a line into the "fname" buffer (which must be at least MAXPATHLEN
- * characters long).
- */
+/* Read a line into the "fname" buffer (which must be at least MAXPATHLEN
+ * characters long). */
int read_filesfrom_line(int fd, char *fname)
{
char ch, *s, *eob = fname + MAXPATHLEN - 1;
if (protocol_version < 29)
return; /* there's nothing we can do */
if (protocol_version >= 30)
- send_msg(MSG_NOOP, "", 0);
+ send_msg(MSG_NOOP, "", 0, 0);
else {
write_int(sock_f_out, cur_flist->count);
write_shortint(sock_f_out, ITEM_IS_NEW);
case MSG_DELETED:
if (msg_bytes >= sizeof line)
goto overflow;
- read_loop(fd, line, msg_bytes);
+#ifdef ICONV_OPTION
+ if (ic_recv != (iconv_t)-1) {
+ ICONV_CONST char *ibuf;
+ char *obuf = line;
+ size_t icnt, ocnt = sizeof line - 1;
+ int add_null = 0;
+ iconv(ic_send, NULL, 0, NULL, 0);
+ while (msg_bytes) {
+ icnt = msg_bytes > sizeof iconv_buf
+ ? sizeof iconv_buf : msg_bytes;
+ read_loop(fd, iconv_buf, icnt);
+ if (!(msg_bytes -= icnt) && !iconv_buf[icnt-1])
+ icnt--, add_null = 1;
+ ibuf = (ICONV_CONST char *)iconv_buf;
+ if (iconv(ic_send, &ibuf,&icnt,
+ &obuf,&ocnt) == (size_t)-1)
+ goto overflow; // XXX
+ }
+ if (add_null)
+ *obuf++ = '\0';
+ msg_bytes = obuf - line;
+ } else
+#endif
+ read_loop(fd, line, msg_bytes);
/* A directory name was sent with the trailing null */
if (msg_bytes > 0 && !line[msg_bytes-1])
log_delete(line, S_IFDIR);
exit_cleanup(RERR_STREAMIO);
}
read_loop(fd, line, msg_bytes);
- rwrite((enum logcode)tag, line, msg_bytes);
+ rwrite((enum logcode)tag, line, msg_bytes, 1);
break;
default:
rprintf(FERROR, "unexpected tag %d [%s]\n",
return;
if (io_multiplexing_out)
- mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt);
+ mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt, 0);
else
writefd_unbuffered(iobuf_f_out, iobuf_out, iobuf_out_cnt);
iobuf_out_cnt = 0;
}
/** Write an message to the multiplexed data stream. */
-int io_multiplex_write(enum msgcode code, const char *buf, size_t len)
+int io_multiplex_write(enum msgcode code, const char *buf, size_t len, int convert)
{
if (!io_multiplexing_out)
return 0;
io_flush(NORMAL_FLUSH);
stats.total_written += (len+4);
- mplex_write(sock_f_out, code, buf, len);
+ mplex_write(sock_f_out, code, buf, len, convert);
return 1;
}
*/
#include "rsync.h"
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
-#include <iconv.h>
-#endif
extern int verbose;
extern int dry_run;
extern int am_daemon;
extern int am_server;
extern int am_sender;
+extern int am_generator;
extern int local_server;
extern int quiet;
extern int module_id;
extern char *stdout_format;
extern char *logfile_format;
extern char *logfile_name;
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
extern iconv_t ic_chck;
#endif
+#ifdef ICONV_OPTION
+extern iconv_t ic_send, ic_recv;
+#endif
extern char curr_dir[];
extern unsigned int module_dirlen;
/* this is the underlying (unformatted) rsync debugging function. Call
* it with FINFO, FERROR or FLOG. Note: recursion can happen with
* certain fatal conditions. */
-void rwrite(enum logcode code, const char *buf, int len)
+void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
{
int trailing_CR_or_NL;
FILE *f = NULL;
+#ifdef ICONV_OPTION
+ iconv_t ic = is_utf8 && ic_recv != (iconv_t)-1 ? ic_recv : ic_chck;
+#else
+#ifdef ICONV_CONST
+ iconv_t ic = ic_chck;
+#endif
+#endif
if (len < 0)
exit_cleanup(RERR_MESSAGEIO);
if (am_server && msg_fd_out >= 0) {
+ assert(!is_utf8);
/* Pass the message to our sibling. */
- send_msg((enum msgcode)code, buf, len);
+ send_msg((enum msgcode)code, buf, len, 0);
return;
}
if (am_server) {
/* Pass the message to the non-server side. */
- if (send_msg((enum msgcode)code, buf, len))
+ if (send_msg((enum msgcode)code, buf, len, !is_utf8))
return;
if (am_daemon) {
/* TODO: can we send the error to the user somehow? */
trailing_CR_or_NL = len && (buf[len-1] == '\n' || buf[len-1] == '\r')
? buf[--len] : 0;
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
-#ifndef ICONV_CONST
-#define ICONV_CONST
-#endif
- if (ic_chck != (iconv_t)-1) {
+#ifdef ICONV_CONST
+ if (ic != (iconv_t)-1) {
char convbuf[1024];
ICONV_CONST char *in_buf = (ICONV_CONST char *)buf;
char *out_buf = convbuf;
size_t in_cnt = len, out_cnt = sizeof convbuf - 1;
- iconv(ic_chck, NULL, 0, NULL, 0);
- while (iconv(ic_chck, &in_buf,&in_cnt,
+ iconv(ic, NULL, 0, NULL, 0);
+ while (iconv(ic, &in_buf,&in_cnt,
&out_buf,&out_cnt) == (size_t)-1) {
if (out_buf != convbuf) {
filtered_fwrite(f, convbuf, out_buf - convbuf, 0);
}
}
- rwrite(code, buf, len);
+ rwrite(code, buf, len, 0);
}
/* This is like rprintf, but it also tries to print some
if (len >= sizeof buf)
exit_cleanup(RERR_MESSAGEIO);
- rwrite(code, buf, len);
+ rwrite(code, buf, len, 0);
}
void rflush(enum logcode code)
p = s + len;
}
- rwrite(code, buf, total);
+ rwrite(code, buf, total, 0);
}
/* Return 1 if the format escape is in the log-format string (e.g. look for
else if (am_server && protocol_version >= 29 && len < MAXPATHLEN) {
if (S_ISDIR(mode))
len++; /* directories include trailing null */
- send_msg(MSG_DELETED, fname, len);
+ send_msg(MSG_DELETED, fname, len, am_generator);
} else {
fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n";
log_formatted(FCLIENT, fmt, "del.", &x.file, fname, &stats,
io_flush(FULL_FLUSH);
handle_stats(f_in);
- send_msg(MSG_DONE, "", 1);
+ send_msg(MSG_DONE, "", 1, 0);
write_varlong(error_pipe[1], stats.total_read, 3);
io_flush(FULL_FLUSH);
io_set_sock_fds(f_in, f_out);
setup_protocol(f_out, f_in);
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
setup_iconv();
#endif
io_set_sock_fds(f_in, f_out);
setup_protocol(f_out,f_in);
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
setup_iconv();
#endif
#define MAX_BATCH_NAME_LEN 256 /* Must be less than MAXPATHLEN-13 */
char *batch_name = NULL;
+#ifdef ICONV_OPTION
+int need_unsorted_flist = 0;
+char *iconv_opt = ICONV_OPTION;
+#endif
+
struct chmod_mode_struct *chmod_modes = NULL;
static int daemon_opt; /* sets am_daemon after option error-reporting */
char const *acls = "no ";
char const *xattrs = "no ";
char const *links = "no ";
+ char const *iconv = "no ";
char const *ipv6 = "no ";
STRUCT_STAT *dumstat;
#ifdef INET6
ipv6 = "";
#endif
+#ifdef ICONV_OPTION
+ iconv = "";
+#endif
rprintf(f, "%s version %s protocol version %d%s\n",
RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
(int)(sizeof (int64) * 8));
rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
got_socketpair, hardlinks, links, ipv6, have_inplace);
- rprintf(f, " %sappend, %sACLs, %sxattrs\n",
- have_inplace, acls, xattrs);
+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv\n",
+ have_inplace, acls, xattrs, iconv);
#ifdef MAINTAINER_MODE
rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
rprintf(F," --only-write-batch=FILE like --write-batch but w/o updating destination\n");
rprintf(F," --read-batch=FILE read a batched update from FILE\n");
rprintf(F," --protocol=NUM force an older protocol version to be used\n");
+#ifdef ICONV_OPTION
+ rprintf(F," --iconv=CONVERT_SPEC request charset conversion of filesnames\n");
+#endif
#ifdef INET6
rprintf(F," -4, --ipv4 prefer IPv4\n");
rprintf(F," -6, --ipv6 prefer IPv6\n");
{"rsh", 'e', POPT_ARG_STRING, &shell_cmd, 0, 0, 0 },
{"rsync-path", 0, POPT_ARG_STRING, &rsync_path, 0, 0, 0 },
{"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
+#ifdef ICONV_OPTION
+ {"iconv", 0, POPT_ARG_STRING, &iconv_opt, 0, 0, 0 },
+#endif
#ifdef INET6
{"ipv4", '4', POPT_ARG_VAL, &default_af_hint, AF_INET, 0, 0 },
{"ipv6", '6', POPT_ARG_VAL, &default_af_hint, AF_INET6, 0, 0 },
if (am_daemon)
set_refuse_options("log-file*");
+#ifdef ICONV_OPTION
+ if (!am_daemon && (arg = getenv("RSYNC_ICONV")) != NULL && *arg)
+ iconv_opt = strdup(arg);
+#endif
+
/* TODO: Call poptReadDefaultConfig; handle errors. */
/* The context leaks in case of an error, but if there's a
long_options, 0);
am_server = 1;
}
+#ifdef ICONV_OPTION
+ iconv_opt = NULL;
+#endif
break;
case OPT_SENDER:
sizeof err_buf);
return 0;
}
+#ifdef ICONV_OPTION
+ iconv_opt = NULL;
+#endif
poptFreeContext(pc);
pc = poptGetContext(RSYNC_NAME, *argc, *argv,
long_daemon_options, 0);
exit_cleanup(0);
}
+#ifdef ICONV_OPTION
+ if (iconv_opt) {
+ if (!am_server && strcmp(iconv_opt, "-") == 0)
+ iconv_opt = NULL;
+ else
+ need_unsorted_flist = 1;
+ }
+#endif
+
#ifndef SUPPORT_LINKS
if (preserve_links && !am_sender) {
snprintf(err_buf, sizeof err_buf,
if (list_only == 1 && !recurse)
argstr[x++] = 'r';
- argstr[x] = '\0';
-
#if SUBPROTOCOL_VERSION != 0
/* If we're speaking a pre-release version of a protocol, we tell
* the server about this by (ab)using the -e option. */
}
#endif
+ argstr[x] = '\0';
+
if (x != 1)
args[ac++] = argstr;
args[ac++] = "--log-format=X";
}
+#ifdef ICONV_OPTION
+ if (iconv_opt) {
+ char *set = strchr(iconv_opt, ',');
+ if (set)
+ set++;
+ else
+ set = iconv_opt;
+ if (asprintf(&arg, "--iconv=%s", set) < 0)
+ goto oom;
+ args[ac++] = arg;
+ }
+#endif
+
if (block_size) {
if (asprintf(&arg, "-B%lu", block_size) < 0)
goto oom;
rprintf(FINFO, "recv_files phase=%d\n", phase);
if (phase == 2 && delay_updates)
handle_delayed_updates(local_name);
- send_msg(MSG_DONE, "", 0);
+ send_msg(MSG_DONE, "", 0, 0);
continue;
}
*/
#include "rsync.h"
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
-#include <iconv.h>
-#endif
#if defined HAVE_LIBCHARSET_H && defined HAVE_LOCALE_CHARSET
#include <libcharset.h>
#elif defined HAVE_LANGINFO_H && defined HAVE_NL_LANGINFO
extern int make_backups;
extern struct file_list *cur_flist, *first_flist, *dir_flist;
extern struct chmod_mode_struct *daemon_chmod_modes;
+#ifdef ICONV_OPTION
+extern char *iconv_opt;
+#endif
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
iconv_t ic_chck = (iconv_t)-1;
+#ifdef ICONV_OPTION
+iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
+int ic_ndx;
+#endif
static const char *default_charset(void)
{
void setup_iconv()
{
+ const char *defset = default_charset();
+# ifdef ICONV_OPTION
+ const char *charset;
+ char *cp;
+#endif
+
if (!am_server && !allow_8bit_chars) {
- const char *defset = default_charset();
/* It's OK if this fails... */
ic_chck = iconv_open(defset, defset);
}
}
}
+
+# ifdef ICONV_OPTION
+ if (!iconv_opt)
+ return;
+
+ if ((cp = strchr(iconv_opt, ',')) != NULL) {
+ if (am_server) /* A local transfer needs this. */
+ iconv_opt = cp + 1;
+ else
+ *cp = '\0';
+ }
+
+ if (!*iconv_opt || (*iconv_opt == '.' && iconv_opt[1] == '\0'))
+ charset = defset;
+ else
+ charset = iconv_opt;
+
+ if ((ic_send = iconv_open(UTF8_CHARSET, charset)) == (iconv_t)-1) {
+ rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n",
+ UTF8_CHARSET, charset);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+
+ if ((ic_recv = iconv_open(charset, UTF8_CHARSET)) == (iconv_t)-1) {
+ rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n",
+ charset, UTF8_CHARSET);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+
+ if (!am_sender)
+ ic_ndx = ++file_extra_cnt;
+
+ if (verbose > 1) {
+ rprintf(FINFO, "%s charset: %s\n",
+ am_server ? "server" : "client",
+ *charset ? charset : "[LOCALE]");
+ }
+# endif
}
#endif
goto invalid_ndx;
if (ndx == NDX_FLIST_EOF) {
flist_eof = 1;
- send_msg(MSG_FLIST_EOF, "", 0);
+ send_msg(MSG_FLIST_EOF, "", 0, 0);
continue;
}
ndx = NDX_FLIST_OFFSET - ndx;
# include <limits.h>
#endif
+#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#include <iconv.h>
+#ifndef ICONV_CONST
+#define ICONV_CONST
+#endif
+#elif defined ICONV_CONST
+#undef ICONV_CONST
+#endif
+
#include <assert.h>
#include "lib/pool_alloc.h"
#define F_GROUP(f) REQ_EXTRA(f, preserve_gid)->unum
#define F_ACL(f) REQ_EXTRA(f, preserve_acls)->num
#define F_XATTR(f) REQ_EXTRA(f, preserve_xattrs)->num
+#define F_NDX(f) REQ_EXTRA(f, ic_ndx)->num
/* These items are per-entry optional and mutally exclusive: */
#define F_HL_GNUM(f) OPT_EXTRA(f, LEN64_BUMP(f))->num
struct file_list {
struct file_list *next, *prev;
- struct file_struct **files;
+ struct file_struct **files, **sorted;
alloc_pool_t file_pool;
int count, malloced;
int low, high; /* 0-relative index values excluding empties */
--only-write-batch=FILE like --write-batch but w/o updating dest
--read-batch=FILE read a batched update from FILE
--protocol=NUM force an older protocol version to be used
+ --iconv=CONVERT_SPEC request charset conversion of filesnames
--checksum-seed=NUM set block/file checksum seed (advanced)
-4, --ipv4 prefer IPv4
-6, --ipv6 prefer IPv6
batch file to force the older protocol version to be used in the batch
file (assuming you can't upgrade the rsync on the reading system).
+dit(bf(--iconv=CONVERT_SPEC)) Rsync can convert filenames between character
+sets using this option. Using a CONVERT_SPEC of "." tells rsync to look up
+the default character-set via the locale setting. Alternately, you can
+fully specify what conversion to do by giving a local and a remote charset
+separated by a comma (local first), e.g. bf(--iconv=utf8,iso88591).
+Finally, you can specify a CONVERT_SPEC of "-" to turn off any conversion.
+The default setting of this option is site-specific, and can also be
+affected via the RSYNC_ICONV environment variable.
+
+Note that rsync does not do any conversion of names in filter files
+(including include/exclude files), in a files-from file, nor those
+specified on the command line. It is up to you to ensure that you're
+requesting the right names from a remote server, and you can specify
+extra include/exclude rules if there are filename differences on the
+two sides that need to be accounted for. (In the future there may be
+a way to specify a UTF-8 filter rule that gets auto-converted to the
+local side's character set.)
+
dit(bf(-4, --ipv4) or bf(-6, --ipv6)) Tells rsync to prefer IPv4/IPv6
when creating sockets. This only affects sockets that rsync has direct
control over, such as the outgoing socket when directly contacting an
dit(bf(CVSIGNORE)) The CVSIGNORE environment variable supplements any
ignore patterns in .cvsignore files. See the bf(--cvs-exclude) option for
more details.
+dit(bf(RSYNC_ICONV)) Specify a default bf(--iconv) setting using this
+environment variable.
dit(bf(RSYNC_RSH)) The RSYNC_RSH environment variable allows you to
override the default shell used as the transport for rsync. Command line
options are permitted after the command name, just as in the bf(-e) option.
* unsuccessful, or if the daemon is being run with -vv. */
for (s = 0; s < ecnt; s++) {
if (!i || verbose > 1)
- rwrite(FLOG, errmsgs[s], strlen(errmsgs[s]));
+ rwrite(FLOG, errmsgs[s], strlen(errmsgs[s]), 0);
free(errmsgs[s]);
}
free(errmsgs);
-e '/^done$/d' \
-e '/ --whole-file$/d' \
-e '/^total: /d' \
+ -e '/^client charset: /d' \
+ -e '/^server charset: /d' \
-e '/^$/,$d' \
<"$outfile" >"$outfile.new"
mv "$outfile.new" "$outfile"
Matt McCutchen's Web Site / rsync / rsync.git / commitdiff