/* -*- c-file-style: "linux" -*-
-
- Copyright (C) 1996-2001 by Andrew Tridgell
- Copyright (C) Paul Mackerras 1996
- Copyright (C) 2001 by Martin Pool <mbp@samba.org>
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-*/
+ *
+ * Copyright (C) 1996-2001 by Andrew Tridgell
+ * Copyright (C) Paul Mackerras 1996
+ * Copyright (C) 2001, 2002 by Martin Pool <mbp@samba.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
-/*
- socket and pipe IO utilities used in rsync
+/**
+ * @file io.c
+ *
+ * Socket and pipe IO utilities used in rsync.
+ *
+ * rsync provides its own multiplexing system, which is used to send
+ * stderr and stdout over a single socket. We need this because
+ * stdout normally carries the binary data stream, and stderr all our
+ * error messages.
+ *
+ * For historical reasons this is off during the start of the
+ * connection, but it's switched on quite early using
+ * io_start_multiplex_out() and io_start_multiplex_in().
+ **/
- tridge, June 1996
- */
#include "rsync.h"
-/* if no timeout is specified then use a 60 second select timeout */
+/** If no timeout is specified then use a 60 second select timeout */
#define SELECT_TIMEOUT 60
static int io_multiplexing_out;
extern struct stats stats;
+const char phase_unknown[] = "unknown";
+
+/**
+ * The connection might be dropped at some point; perhaps because the
+ * remote instance crashed. Just giving the offset on the stream is
+ * not very helpful. So instead we try to make io_phase_name point to
+ * something useful.
+ *
+ * For buffered/multiplexed IO these names will be somewhat
+ * approximate; perhaps for ease of support we would rather make the
+ * buffer always flush when a single application-level IO finishes.
+ *
+ * @todo Perhaps we want some simple stack functionality, but there's
+ * no need to overdo it.
+ **/
+const char *io_write_phase = phase_unknown;
+const char *io_read_phase = phase_unknown;
+
/** Ignore EOF errors while reading a module listing if the remote
version is 24 or less. */
int kludge_around_eof = False;
static int io_error_fd = -1;
+static int io_filesfrom_f_in = -1;
+static int io_filesfrom_f_out = -1;
+static char io_filesfrom_buf[2048];
+static char *io_filesfrom_bp;
+static char io_filesfrom_lastchar;
+static int io_filesfrom_buflen;
static void read_loop(int fd, char *buf, size_t len);
}
}
-/* setup the fd used to propogate errors */
+/** Setup the fd used to propagate errors */
void io_set_error_fd(int fd)
{
io_error_fd = fd;
}
-/* read some data from the error fd and write it to the write log code */
+/** Read some data from the error fd and write it to the write log code */
static void read_error_fd(void)
{
char buf[200];
- int n;
+ size_t n;
int fd = io_error_fd;
int tag, len;
- /* io_error_fd is temporarily disabled -- is this meant to
- * prevent indefinite recursion? */
+ /* io_error_fd is temporarily disabled -- is this meant to
+ * prevent indefinite recursion? */
io_error_fd = -1;
read_loop(fd, buf, 4);
while (len) {
n = len;
- if (n > (sizeof(buf)-1)) n = sizeof(buf)-1;
+ if (n > (sizeof(buf)-1))
+ n = sizeof(buf)-1;
read_loop(fd, buf, n);
rwrite((enum logcode)tag, buf, n);
len -= n;
io_error_fd = fd;
}
+/**
+ * When we're the receiver and we have a local --files-from list of names
+ * that needs to be sent over the socket to the sender, we have to do two
+ * things at the same time: send the sender a list of what files we're
+ * processing and read the incoming file+info list from the sender. We do
+ * this by augmenting the read_timeout() function to copy this data. It
+ * uses the io_filesfrom_buf to read a block of data from f_in (when it is
+ * ready, since it might be a pipe) and then blast it out f_out (when it
+ * is ready to receive more data).
+ */
+void io_set_filesfrom_fds(int f_in, int f_out)
+{
+ io_filesfrom_f_in = f_in;
+ io_filesfrom_f_out = f_out;
+ io_filesfrom_bp = io_filesfrom_buf;
+ io_filesfrom_lastchar = '\0';
+ io_filesfrom_buflen = 0;
+}
+/**
+ * It's almost always an error to get an EOF when we're trying to read
+ * from the network, because the protocol is self-terminating.
+ *
+ * However, there is one unfortunate cases where it is not, which is
+ * rsync <2.4.6 sending a list of modules on a server, since the list
+ * is terminated by closing the socket. So, for the section of the
+ * program where that is a problem (start_socket_client),
+ * kludge_around_eof is True and we just exit.
+ */
static void whine_about_eof (void)
{
- /**
- It's almost always an error to get an EOF when we're trying
- to read from the network, because the protocol is
- self-terminating.
-
- However, there is one unfortunate cases where it is not,
- which is rsync <2.4.6 sending a list of modules on a
- server, since the list is terminated by closing the socket.
- So, for the section of the program where that is a problem
- (start_socket_client), kludge_around_eof is True and we
- just exit.
- */
-
if (kludge_around_eof)
exit_cleanup (0);
else {
}
-/*!
+/**
* Read from a socket with IO timeout. return the number of bytes
* read. If no bytes can be read then exit, never return a number <= 0.
*
while (ret == 0) {
/* until we manage to read *something* */
- fd_set fds;
+ fd_set r_fds, w_fds;
struct timeval tv;
int fd_count = fd+1;
int count;
- FD_ZERO(&fds);
- FD_SET(fd, &fds);
+ FD_ZERO(&r_fds);
+ FD_SET(fd, &r_fds);
if (io_error_fd != -1) {
- FD_SET(io_error_fd, &fds);
- if (io_error_fd > fd) fd_count = io_error_fd+1;
+ FD_SET(io_error_fd, &r_fds);
+ if (io_error_fd >= fd_count) fd_count = io_error_fd+1;
+ }
+ if (io_filesfrom_f_out != -1) {
+ int new_fd;
+ if (io_filesfrom_buflen == 0) {
+ if (io_filesfrom_f_in != -1) {
+ FD_SET(io_filesfrom_f_in, &r_fds);
+ new_fd = io_filesfrom_f_in;
+ } else {
+ io_filesfrom_f_out = -1;
+ new_fd = -1;
+ }
+ } else {
+ FD_ZERO(&w_fds);
+ FD_SET(io_filesfrom_f_out, &w_fds);
+ new_fd = io_filesfrom_f_out;
+ }
+ if (new_fd >= fd_count) fd_count = new_fd+1;
}
tv.tv_sec = io_timeout?io_timeout:SELECT_TIMEOUT;
errno = 0;
- count = select(fd_count, &fds, NULL, NULL, &tv);
+ count = select(fd_count, &r_fds,
+ io_filesfrom_buflen? &w_fds : NULL,
+ NULL, &tv);
if (count == 0) {
check_timeout();
continue;
}
- if (io_error_fd != -1 && FD_ISSET(io_error_fd, &fds)) {
+
+ if (io_error_fd != -1 && FD_ISSET(io_error_fd, &r_fds)) {
read_error_fd();
}
- if (!FD_ISSET(fd, &fds)) continue;
+ if (io_filesfrom_f_out != -1) {
+ if (io_filesfrom_buflen) {
+ if (FD_ISSET(io_filesfrom_f_out, &w_fds)) {
+ int l = write(io_filesfrom_f_out,
+ io_filesfrom_bp,
+ io_filesfrom_buflen);
+ if (l > 0) {
+ if (!(io_filesfrom_buflen -= l))
+ io_filesfrom_bp = io_filesfrom_buf;
+ else
+ io_filesfrom_bp += l;
+ } else {
+ /* XXX should we complain? */
+ io_filesfrom_f_out = -1;
+ }
+ }
+ } else if (io_filesfrom_f_in != -1) {
+ if (FD_ISSET(io_filesfrom_f_in, &r_fds)) {
+ int l = read(io_filesfrom_f_in,
+ io_filesfrom_buf,
+ sizeof io_filesfrom_buf);
+ if (l <= 0) {
+ /* Send end-of-file marker */
+ io_filesfrom_buf[0] = '\0';
+ io_filesfrom_buf[1] = '\0';
+ io_filesfrom_buflen = io_filesfrom_lastchar? 2 : 1;
+ io_filesfrom_f_in = -1;
+ } else {
+ extern int eol_nulls;
+ if (!eol_nulls) {
+ char *s = io_filesfrom_buf + l;
+ /* Transform CR and/or LF into '\0' */
+ while (s-- > io_filesfrom_buf) {
+ if (*s == '\n' || *s == '\r')
+ *s = '\0';
+ }
+ }
+ if (!io_filesfrom_lastchar) {
+ /* Last buf ended with a '\0', so don't
+ * let this buf start with one. */
+ while (l && !*io_filesfrom_bp)
+ io_filesfrom_bp++, l--;
+ }
+ if (!l)
+ io_filesfrom_bp = io_filesfrom_buf;
+ else {
+ char *f = io_filesfrom_bp;
+ char *t = f;
+ char *eob = f + l;
+ /* Eliminate any multi-'\0' runs. */
+ while (f != eob) {
+ if (!(*t++ = *f++)) {
+ while (f != eob && !*f)
+ f++, l--;
+ }
+ }
+ io_filesfrom_lastchar = f[-1];
+ }
+ io_filesfrom_buflen = l;
+ }
+ }
+ }
+ }
+
+ if (!FD_ISSET(fd, &r_fds)) continue;
n = read(fd, buf, len);
return ret;
}
+/**
+ * Read a line into the "fname" buffer (which must be at least MAXPATHLEN
+ * characters long).
+ */
+int read_filesfrom_line(int fd, char *fname)
+{
+ char ch, *s, *eob = fname + MAXPATHLEN - 1;
+ int cnt;
+ extern int io_timeout;
+ extern int eol_nulls;
+ extern char *remote_filesfrom_file;
+ extern int am_server;
+ int reading_remotely = remote_filesfrom_file || (am_server && fd == 0);
+ int nulls = eol_nulls || reading_remotely;
+
+ start:
+ s = fname;
+ while (1) {
+ cnt = read(fd, &ch, 1);
+ if (cnt < 0 && (errno == EWOULDBLOCK
+ || errno == EINTR || errno == EAGAIN)) {
+ struct timeval tv;
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(fd, &fds);
+ tv.tv_sec = io_timeout? io_timeout : SELECT_TIMEOUT;
+ tv.tv_usec = 0;
+ if (!select(fd+1, &fds, NULL, NULL, &tv))
+ check_timeout();
+ continue;
+ }
+ if (cnt != 1)
+ break;
+ if (nulls? !ch : (ch == '\r' || ch == '\n')) {
+ /* Skip empty lines if reading locally. */
+ if (!reading_remotely && s == fname)
+ continue;
+ break;
+ }
+ if (s < eob)
+ *s++ = ch;
+ }
+ *s = '\0';
+
+ /* Dump comments. */
+ if (*fname == '#' || *fname == ';')
+ goto start;
+ return s - fname;
+}
-/*! Continue trying to read len bytes - don't return until len has
- been read. */
+/**
+ * Continue trying to read len bytes - don't return until len has been
+ * read.
+ **/
static void read_loop (int fd, char *buf, size_t len)
{
while (len) {
*/
static int read_unbuffered(int fd, char *buf, size_t len)
{
- static int remaining;
+ static size_t remaining;
int tag, ret = 0;
char line[1024];
}
if (remaining > sizeof(line) - 1) {
- rprintf(FERROR, "multiplexing overflow %d\n\n",
- remaining);
+ rprintf(FERROR, "multiplexing overflow %ld\n\n",
+ (long)remaining);
exit_cleanup(RERR_STREAMIO);
}
-/* do a buffered read from fd. don't return until all N bytes
- have been read. If all N can't be read then exit with an error */
+/**
+ * Do a buffered read from @p fd. Don't return until all @p n bytes
+ * have been read. If all @p n can't be read then exit with an
+ * error.
+ **/
static void readfd (int fd, char *buffer, size_t N)
{
int ret;
- int total=0;
+ size_t total=0;
while (total < N) {
io_flush();
int64 read_longint(int f)
{
- extern int remote_version;
int64 ret;
char b[8];
ret = read_int(f);
rprintf(FERROR,"Integer overflow - attempted 64 bit offset\n");
exit_cleanup(RERR_UNSUPPORTED);
#else
- if (remote_version >= 16) {
- readfd(f,b,8);
- ret = IVAL(b,0) | (((int64)IVAL(b,4))<<32);
- }
+ readfd(f,b,8);
+ ret = IVAL(b,0) | (((int64)IVAL(b,4))<<32);
#endif
return ret;
return c;
}
-/* write len bytes to fd */
+
+/**
+ * Sleep after writing to limit I/O bandwidth usage.
+ *
+ * @todo Rather than sleeping after each write, it might be better to
+ * use some kind of averaging. The current algorithm seems to always
+ * use a bit less bandwidth than specified, because it doesn't make up
+ * for slow periods. But arguably this is a feature. In addition, we
+ * ought to take the time used to write the data into account.
+ **/
+static void sleep_for_bwlimit(int bytes_written)
+{
+ struct timeval tv;
+
+ if (!bwlimit)
+ return;
+
+ assert(bytes_written > 0);
+ assert(bwlimit > 0);
+
+ tv.tv_usec = bytes_written * 1000 / bwlimit;
+ tv.tv_sec = tv.tv_usec / 1000000;
+ tv.tv_usec = tv.tv_usec % 1000000;
+
+ select(0, NULL, NULL, NULL, &tv);
+}
+
+
+/**
+ * Write len bytes to the file descriptor @p fd.
+ *
+ * This function underlies the multiplexing system. The body of the
+ * application never calls this function directly.
+ **/
static void writefd_unbuffered(int fd,char *buf,size_t len)
{
- int total = 0;
+ size_t total = 0;
fd_set w_fds, r_fds;
int fd_count, count;
struct timeval tv;
while (total < len) {
FD_ZERO(&w_fds);
- FD_ZERO(&r_fds);
FD_SET(fd,&w_fds);
fd_count = fd;
if (io_error_fd != -1) {
+ FD_ZERO(&r_fds);
FD_SET(io_error_fd,&r_fds);
if (io_error_fd > fd_count)
fd_count = io_error_fd;
}
if (FD_ISSET(fd, &w_fds)) {
- int ret, n = len-total;
+ int ret;
+ size_t n = len-total;
ret = write(fd,buf+total,n);
if (ret == -1 && errno == EINTR) {
}
if (ret <= 0) {
- rprintf(FERROR,
- "error writing %d unbuffered bytes"
- " - exiting: %s\n", len,
+ /* Don't try to write errors back
+ * across the stream */
+ io_multiplexing_close();
+ rprintf(FERROR, RSYNC_NAME
+ ": writefd_unbuffered failed to write %ld bytes: phase \"%s\": %s\n",
+ (long) len, io_write_phase,
strerror(errno));
exit_cleanup(RERR_STREAMIO);
}
- /* Sleep after writing to limit I/O bandwidth */
- if (bwlimit)
- {
- tv.tv_sec = 0;
- tv.tv_usec = ret * 1000 / bwlimit;
- while (tv.tv_usec > 1000000)
- {
- tv.tv_sec++;
- tv.tv_usec -= 1000000;
- }
- select(0, NULL, NULL, NULL, &tv);
- }
+ sleep_for_bwlimit(ret);
total += ret;
io_buffer_count = 0;
}
-/* write an message to a multiplexed stream. If this fails then rsync
- exits */
+/**
+ * Write an message to a multiplexed stream. If this fails then rsync
+ * exits.
+ **/
static void mplex_write(int fd, enum logcode code, char *buf, size_t len)
{
char buffer[4096];
- int n = len;
+ size_t n = len;
SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len);
}
-/* XXX: fd is ignored, which seems a little strange. */
-void io_end_buffering(int fd)
+void io_end_buffering(void)
{
io_flush();
if (!io_multiplexing_out) {
}
while (len) {
- int n = MIN(len, IO_BUFFER_SIZE-io_buffer_count);
+ int n = MIN((int) len, IO_BUFFER_SIZE-io_buffer_count);
if (n > 0) {
memcpy(io_buffer+io_buffer_count, buf, n);
buf += n;
}
+void write_int_named(int f, int32 x, const char *phase)
+{
+ io_write_phase = phase;
+ write_int(f, x);
+ io_write_phase = phase_unknown;
+}
+
+
/*
* Note: int64 may actually be a 32-bit type if ./configure couldn't find any
* 64-bit types on this platform.
*/
void write_longint(int f, int64 x)
{
- extern int remote_version;
char b[8];
- if (remote_version < 16 || x <= 0x7FFFFFFF) {
+ if (x <= 0x7FFFFFFF) {
write_int(f, (int)x);
return;
}
+#ifdef NO_INT64
+ rprintf(FERROR,"Integer overflow - attempted 64 bit offset\n");
+ exit_cleanup(RERR_UNSUPPORTED);
+#else
write_int(f, (int32)0xFFFFFFFF);
SIVAL(b,0,(x&0xFFFFFFFF));
SIVAL(b,4,((x>>32)&0xFFFFFFFF));
writefd(f,b,8);
+#endif
}
void write_buf(int f,char *buf,size_t len)
writefd(f,buf,len);
}
-/* write a string to the connection */
+/** Write a string to the connection */
static void write_sbuf(int f,char *buf)
{
write_buf(f, buf, strlen(buf));
+/**
+ * Read a line of up to @p maxlen characters into @p buf. Does not
+ * contain a trailing newline or carriage return.
+ *
+ * @return 1 for success; 0 for io error or truncation.
+ **/
int read_line(int f, char *buf, size_t maxlen)
{
while (maxlen) {
buf[0] = 0;
read_buf(f, buf, 1);
- if (buf[0] == 0) return 0;
+ if (buf[0] == 0)
+ return 0;
if (buf[0] == '\n') {
buf[0] = 0;
break;
}
-/* setup for multiplexing an error stream with the data stream */
+/** Setup for multiplexing an error stream with the data stream */
void io_start_multiplex_out(int fd)
{
multiplex_out_fd = fd;
io_multiplexing_out = 1;
}
-/* setup for multiplexing an error stream with the data stream */
+/** Setup for multiplexing an error stream with the data stream */
void io_start_multiplex_in(int fd)
{
multiplex_in_fd = fd;
io_multiplexing_in = 1;
}
-/* write an message to the multiplexed error stream */
+/** Write an message to the multiplexed error stream */
int io_multiplex_write(enum logcode code, char *buf, size_t len)
{
if (!io_multiplexing_out) return 0;
return 1;
}
-/* stop output multiplexing */
+/** Stop output multiplexing */
void io_multiplexing_close(void)
{
io_multiplexing_out = 0;