From bc3fcf1d39ca698386e09f1abc1a54178102e558 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 3 Apr 2008 23:56:49 -0700 Subject: [PATCH] The patches for 3.0.1 and 3.0.2. --- atimes.diff | 14 +- backup-dir-dels.diff | 2 +- checksum-reading.diff | 18 +-- checksum-updating.diff | 10 +- checksum-xattr.diff | 2 +- crtimes.diff | 14 +- db.diff | 325 ++++++++++++++++++++++++++++++++--------- detect-renamed.diff | 2 +- fileflags.diff | 14 +- ignore-case.diff | 4 +- link-by-hash.diff | 2 +- slow-down.diff | 2 +- transliterate.diff | 4 +- usermap.diff | 6 +- 14 files changed, 303 insertions(+), 116 deletions(-) diff --git a/atimes.diff b/atimes.diff index 462a40b..cdd41c3 100644 --- a/atimes.diff +++ b/atimes.diff @@ -44,7 +44,7 @@ diff --git a/flist.c b/flist.c extern int relative_paths; extern int implied_dirs; extern int file_extra_cnt; -@@ -384,7 +385,7 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen) +@@ -388,7 +389,7 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen) static void send_file_entry(int f, const char *fname, struct file_struct *file, int ndx, int first_ndx) { @@ -53,7 +53,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; -@@ -458,6 +459,13 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -462,6 +463,13 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, xflags |= XMIT_SAME_TIME; else modtime = file->modtime; @@ -67,7 +67,7 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS if (tmp_dev != 0) { -@@ -529,6 +537,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -533,6 +541,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, } if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); @@ -76,7 +76,7 @@ diff --git a/flist.c b/flist.c if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) write_int(f, uid); -@@ -615,7 +625,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -619,7 +629,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { @@ -85,7 +85,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; -@@ -749,6 +759,16 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -753,6 +763,16 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } if (!(xflags & XMIT_SAME_MODE)) mode = from_wire_mode(read_int(f)); @@ -102,7 +102,7 @@ diff --git a/flist.c b/flist.c if (chmod_modes && !S_ISLNK(mode)) mode = tweak_mode(mode, chmod_modes); -@@ -878,6 +898,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -882,6 +902,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, F_GROUP(file) = gid; file->flags |= gid_flags; } @@ -111,7 +111,7 @@ diff --git a/flist.c b/flist.c if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; -@@ -1226,6 +1248,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1230,6 +1252,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, F_OWNER(file) = st.st_uid; if (gid_ndx) /* Check gid_ndx instead of preserve_gid for del support */ F_GROUP(file) = st.st_gid; diff --git a/backup-dir-dels.diff b/backup-dir-dels.diff index 51d8489..82b6adc 100644 --- a/backup-dir-dels.diff +++ b/backup-dir-dels.diff @@ -215,7 +215,7 @@ diff --git a/options.c b/options.c + if (!*backup_dir_dels) + goto options_rejected; + clean_fname(backup_dir_dels, 1); -+ if (check_filter(elp, backup_dir_dels, 1) < 0) ++ if (check_filter(elp, FLOG, backup_dir_dels, 1) < 0) + goto options_rejected; + } } diff --git a/checksum-reading.diff b/checksum-reading.diff index 159c7ad..0e1ba3a 100644 --- a/checksum-reading.diff +++ b/checksum-reading.diff @@ -342,7 +342,7 @@ diff --git a/flist.c b/flist.c /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's * F_PATHNAME(), or (2) "NULL, dir, dirlen" to chdir() to the supplied dir, * with dir == NULL taken to be the starting directory, and dirlen < 0 -@@ -1010,7 +1255,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1014,7 +1259,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, STRUCT_STAT *stp, int flags, int filter_level) { static char *lastdir; @@ -351,7 +351,7 @@ diff --git a/flist.c b/flist.c struct file_struct *file; char thisname[MAXPATHLEN]; char linkname[MAXPATHLEN]; -@@ -1160,9 +1405,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1164,9 +1409,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(lastdir, thisname, len); lastdir[len] = '\0'; lastdir_len = len; @@ -369,7 +369,7 @@ diff --git a/flist.c b/flist.c basename_len = strlen(basename) + 1; /* count the '\0' */ #ifdef SUPPORT_LINKS -@@ -1235,14 +1487,18 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1239,14 +1491,18 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(bp + basename_len, linkname, linkname_len); #endif @@ -391,7 +391,7 @@ diff --git a/flist.c b/flist.c /* This code is only used by the receiver when it is building * a list of files for a delete pass. */ if (keep_dirlinks && linkname_len && flist) { -@@ -2155,7 +2411,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) +@@ -2159,7 +2415,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) * file-list to check if this is a 1-file xfer. */ send_extra_file_list(f, 1); } @@ -401,7 +401,7 @@ diff --git a/flist.c b/flist.c return flist; } -@@ -2257,7 +2514,7 @@ struct file_list *recv_file_list(int f) +@@ -2261,7 +2518,7 @@ struct file_list *recv_file_list(int f) else if (f >= 0) recv_id_list(f, flist); @@ -410,7 +410,7 @@ diff --git a/flist.c b/flist.c if (protocol_version < 30) { /* Recv the io_error flag */ -@@ -2455,7 +2712,7 @@ void flist_free(struct file_list *flist) +@@ -2459,7 +2716,7 @@ void flist_free(struct file_list *flist) /* This routine ensures we don't have any duplicate names in our file list. * duplicate names can cause corruption because of the pipelining. */ @@ -419,7 +419,7 @@ diff --git a/flist.c b/flist.c { char fbuf[MAXPATHLEN]; int i, prev_i; -@@ -2506,7 +2763,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) +@@ -2510,7 +2767,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) /* If one is a dir and the other is not, we want to * keep the dir because it might have contents in the * list. Otherwise keep the first one. */ @@ -428,7 +428,7 @@ diff --git a/flist.c b/flist.c struct file_struct *fp = flist->sorted[j]; if (!S_ISDIR(fp->mode)) keep = i, drop = j; -@@ -2522,8 +2779,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) +@@ -2526,8 +2783,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) } else keep = j, drop = i; @@ -439,7 +439,7 @@ diff --git a/flist.c b/flist.c rprintf(FINFO, "removing duplicate name %s from file list (%d)\n", f_name(file, fbuf), drop + flist->ndx_start); -@@ -2545,7 +2802,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) +@@ -2549,7 +2806,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) } flist->high = prev_i; diff --git a/checksum-updating.diff b/checksum-updating.diff index 20e9bcd..1fbfc30 100644 --- a/checksum-updating.diff +++ b/checksum-updating.diff @@ -389,7 +389,7 @@ diff --git a/flist.c b/flist.c } /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's -@@ -1360,6 +1571,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1364,6 +1575,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, if (excl_ret) { if (ignore_perishable) non_perishable_cnt++; @@ -398,7 +398,7 @@ diff --git a/flist.c b/flist.c return NULL; } -@@ -1406,13 +1619,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1410,13 +1623,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, lastdir[len] = '\0'; lastdir_len = len; if (checksum_files && am_sender && flist) @@ -414,7 +414,7 @@ diff --git a/flist.c b/flist.c } } basename_len = strlen(basename) + 1; /* count the '\0' */ -@@ -1494,7 +1707,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1498,7 +1711,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, if (always_checksum && am_sender && S_ISREG(st.st_mode)) { if (flist && checksum_files) @@ -423,7 +423,7 @@ diff --git a/flist.c b/flist.c else file_checksum(thisname, st.st_size, tmp_sum); } -@@ -1820,6 +2033,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, +@@ -1824,6 +2037,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, closedir(d); @@ -433,7 +433,7 @@ diff --git a/flist.c b/flist.c if (f >= 0 && recurse && !divert_dirs) { int i, end = flist->used - 1; /* send_if_directory() bumps flist->used, so use "end". */ -@@ -2413,6 +2629,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) +@@ -2417,6 +2633,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) } } else flist_eof = 1; diff --git a/checksum-xattr.diff b/checksum-xattr.diff index 88c2e94..7068ab7 100644 --- a/checksum-xattr.diff +++ b/checksum-xattr.diff @@ -11,7 +11,7 @@ To use this patch, run these commands for a successful build: diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -1235,7 +1235,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1239,7 +1239,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(bp + basename_len, linkname, linkname_len); #endif diff --git a/crtimes.diff b/crtimes.diff index 1d50bc9..c5b35dd 100644 --- a/crtimes.diff +++ b/crtimes.diff @@ -48,7 +48,7 @@ diff --git a/flist.c b/flist.c extern int relative_paths; extern int implied_dirs; extern int file_extra_cnt; -@@ -385,7 +386,7 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen) +@@ -389,7 +390,7 @@ int change_pathname(struct file_struct *file, const char *dir, int dirlen) static void send_file_entry(int f, const char *fname, struct file_struct *file, int ndx, int first_ndx) { @@ -57,7 +57,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_FILEFLAGS static uint32 fileflags; -@@ -470,6 +471,13 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -474,6 +475,13 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, xflags |= XMIT_SAME_TIME; else modtime = file->modtime; @@ -71,7 +71,7 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS if (tmp_dev != 0) { -@@ -539,6 +547,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -543,6 +551,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, else write_int(f, modtime); } @@ -80,7 +80,7 @@ diff --git a/flist.c b/flist.c if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); #ifdef SUPPORT_FILEFLAGS -@@ -631,7 +641,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -635,7 +645,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { @@ -89,7 +89,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_FILEFLAGS static uint32 fileflags; -@@ -766,6 +776,19 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -770,6 +780,19 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } else modtime = read_int(f); } @@ -109,7 +109,7 @@ diff --git a/flist.c b/flist.c if (!(xflags & XMIT_SAME_MODE)) mode = from_wire_mode(read_int(f)); -@@ -905,6 +928,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -909,6 +932,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, F_GROUP(file) = gid; file->flags |= gid_flags; } @@ -118,7 +118,7 @@ diff --git a/flist.c b/flist.c if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; -@@ -1257,6 +1282,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1261,6 +1286,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, F_OWNER(file) = st.st_uid; if (gid_ndx) /* Check gid_ndx instead of preserve_gid for del support */ F_GROUP(file) = st.st_gid; diff --git a/db.diff b/db.diff index 6df344c..4ee3915 100644 --- a/db.diff +++ b/db.diff @@ -3,9 +3,14 @@ about the files it is dealing with. This adds both the --db=CONFIG_FILE option and the "db config" daemon parameter. For the moment this only adds checksum caching when the --checksum option -is used. Future improvments may include: +is used. Future improvements may include: - Updating of MD5 checksums when transferring any file, even w/o -c. + We should be able to extend this to work for MD4 checksums too if we + make the sender force checksum_seed to 0 when using a DB and having + the receiving side check to see if it got a 0 checksum_seed. (We + probably don't want to compute 2 MD4 checksums for the case where + the checksum_seed is non-zero.) - Caching of path info that allows for the finding of files to use for moving/linking/copying/alternate-basis-use. @@ -101,15 +106,16 @@ diff --git a/cleanup.c b/cleanup.c diff --git a/clientserver.c b/clientserver.c --- a/clientserver.c +++ b/clientserver.c -@@ -42,6 +42,7 @@ extern int numeric_ids; +@@ -42,13 +42,16 @@ extern int numeric_ids; extern int filesfrom_fd; extern int remote_protocol; extern int protocol_version; +extern int always_checksum; extern int io_timeout; extern int no_detach; ++extern int use_db; extern int write_batch; -@@ -49,6 +50,7 @@ extern int default_af_hint; + extern int default_af_hint; extern int logfile_format_has_i; extern int logfile_format_has_o_or_i; extern mode_t orig_umask; @@ -117,27 +123,65 @@ diff --git a/clientserver.c b/clientserver.c extern char *bind_address; extern char *sockopts; extern char *config_file; -@@ -782,6 +784,12 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) - } else if (am_root < 0) /* Treat --fake-super from client as --super. */ - am_root = 2; +@@ -554,6 +557,9 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) -+ db_config = lp_db_config(i); -+ if (!*db_config || (!always_checksum && protocol_version < 30)) -+ db_config = NULL; -+ else -+ db_read_config(FLOG, db_config); + log_init(1); + ++ if (*lp_db_config(i)) ++ db_read_config(FLOG, lp_db_config(i)); + - if (filesfrom_fd == 0) - filesfrom_fd = f_in; + #ifdef HAVE_PUTENV + if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) { + char *modname, *modpath, *hostaddr, *hostname, *username; +@@ -770,6 +776,10 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) + am_server = 1; /* Don't let someone try to be tricky. */ + quiet = 0; ++ db_config = NULL; ++ if (!always_checksum) ++ use_db = 0; ++ + if (lp_ignore_errors(module_id)) + ignore_errors = 1; + if (write_batch < 0) diff --git a/configure.in b/configure.in --- a/configure.in +++ b/configure.in -@@ -969,6 +969,8 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x" +@@ -314,7 +314,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \ + sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \ + netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \ + sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \ +- popt.h popt/popt.h) ++ popt.h popt/popt.h mysql/mysql.h sqlite3.h) + AC_HEADER_MAJOR + + AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[ +@@ -969,6 +969,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x" fi fi -+LIBS="$LIBS -lmysqlclient -lsqlite3" ++AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0) ++if test x$MYSQL_CONFIG = x1; then ++ AC_MSG_CHECKING(for mysql version >= 4) ++ mysql_version=`mysql_config --version` ++ mysql_major_version=`echo $mysql_version | sed 's/\..*//'` ++ if test $mysql_major_version -lt 4; then ++ AC_MSG_RESULT(no.. skipping MySQL) ++ else ++ AC_MSG_RESULT(yes) ++ ++ MYSQL_CFLAGS=`mysql_config --cflags` ++ MYSQL_LIBS=`mysql_config --libs` ++ ++ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS" ++ LIBS="$MYSQL_LIBS $LIBS" ++ ++ AC_CHECK_LIB(mysqlclient, mysql_init) ++ fi ++fi ++ ++AC_CHECK_LIB(sqlite3, sqlite3_open) ++AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2) + case "$CC" in ' checker'*|checker*) @@ -146,7 +190,7 @@ diff --git a/db.c b/db.c new file mode 100644 --- /dev/null +++ b/db.c -@@ -0,0 +1,557 @@ +@@ -0,0 +1,566 @@ +/* + * Routines to access extended file info via DB. + * @@ -169,15 +213,22 @@ new file mode 100644 +#include "rsync.h" +#include "ifuncs.h" + ++#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT +#define USE_MYSQL -+#define USE_SQLITE -+ -+#ifdef USE_MYSQL +#include +#include +#endif -+#ifdef USE_SQLITE ++ ++#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3 ++#define USE_SQLITE +#include ++#ifndef HAVE_SQLITE3_OPEN_V2 ++#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \ ++ sqlite3_open(dbname, dbhptr) ++#endif ++#ifndef HAVE_SQLITE3_PREPARE_V2 ++#define sqlite3_prepare_v2 sqlite3_prepare ++#endif +#endif + +extern int protocol_version; @@ -220,9 +271,12 @@ new file mode 100644 +static int md_num; +static enum logcode log_code; + ++#ifdef USE_MYSQL +static unsigned int bind_disk_id; +static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime; -+static char bind_thishost[256], bind_sum[MAX_DIGEST_LEN]; ++static char bind_sum[MAX_DIGEST_LEN]; ++#endif ++static char bind_thishost[256]; +static int bind_thishost_len; + +static unsigned int prior_disk_id = 0; @@ -425,13 +479,8 @@ new file mode 100644 +{ + char *sql; + -+#if 0 + if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0) + return 0; -+#else -+ if (sqlite3_open(dbname, &dbh.sqlite) != 0) -+ return 0; -+#endif + + sql = "SELECT disk_id" + " FROM disk" @@ -527,6 +576,7 @@ new file mode 100644 + dbh.all = NULL; +} + ++#ifdef USE_MYSQL +static MYSQL_STMT *exec_mysql(int ndx) +{ + MYSQL_STMT *stmt = statements[ndx].mysql; @@ -546,7 +596,9 @@ new file mode 100644 + + return stmt; +} ++#endif + ++#ifdef USE_MYSQL +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx) +{ + unsigned long length[32]; @@ -580,6 +632,7 @@ new file mode 100644 + + return is_null[0] ? 0 : 1; +} ++#endif + +static void get_disk_id(unsigned long long devno) +{ @@ -715,7 +768,7 @@ diff --git a/flist.c b/flist.c extern int eol_nulls; extern int relative_paths; extern int implied_dirs; -@@ -1235,14 +1236,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1239,14 +1240,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(bp + basename_len, linkname, linkname_len); #endif @@ -735,7 +788,7 @@ diff --git a/flist.c b/flist.c /* This code is only used by the receiver when it is building * a list of files for a delete pass. */ if (keep_dirlinks && linkname_len && flist) { -@@ -1858,6 +1861,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) +@@ -1862,6 +1865,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0); int implied_dot_dir = 0; @@ -834,7 +887,7 @@ diff --git a/main.c b/main.c exit_cleanup(RERR_SYNTAX); } -+ if (db_config && (always_checksum || protocol_version >= 30)) ++ if (db_config && always_checksum) + db_read_config(FERROR, db_config); + if (am_server) { @@ -851,15 +904,46 @@ diff --git a/options.c b/options.c int eol_nulls = 0; int protect_args = 0; int human_readable = 0; -@@ -321,6 +322,7 @@ void usage(enum logcode F) +@@ -229,6 +230,7 @@ static void print_rsync_version(enum logcode f) + char const *links = "no "; + char const *iconv = "no "; + char const *ipv6 = "no "; ++ char const *db = "no "; + STRUCT_STAT *dumstat; + + #if SUBPROTOCOL_VERSION != 0 +@@ -261,6 +263,11 @@ static void print_rsync_version(enum logcode f) + #if defined HAVE_LUTIMES && defined HAVE_UTIMES + symtimes = ""; + #endif ++#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT ++ db = ""; ++#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3 ++ db = ""; ++#endif + + rprintf(f, "%s version %s protocol version %d%s\n", + RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); +@@ -274,8 +281,8 @@ static void print_rsync_version(enum logcode f) + (int)(sizeof (int64) * 8)); + rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", + got_socketpair, hardlinks, links, ipv6, have_inplace); +- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", +- have_inplace, acls, xattrs, iconv, symtimes); ++ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n", ++ have_inplace, acls, xattrs, iconv, symtimes, db); + + #ifdef MAINTAINER_MODE + rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); +@@ -321,6 +328,7 @@ void usage(enum logcode F) rprintf(F," -q, --quiet suppress non-error messages\n"); rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); -+ rprintf(F," --db=CONFIG_FILE specify a config file for FS DB\n"); ++ rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n"); rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); rprintf(F," -r, --recursive recurse into directories\n"); -@@ -579,6 +581,7 @@ static struct poptOption long_options[] = { +@@ -579,6 +587,7 @@ static struct poptOption long_options[] = { {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, @@ -870,18 +954,17 @@ diff --git a/options.c b/options.c diff --git a/pipe.c b/pipe.c --- a/pipe.c +++ b/pipe.c -@@ -26,6 +26,10 @@ extern int am_sender; +@@ -26,6 +26,9 @@ extern int am_sender; extern int am_server; extern int blocking_io; extern int filesfrom_fd; +extern int always_checksum; -+extern int protocol_version; +extern int use_db; +extern char *db_config; extern mode_t orig_umask; extern char *logfile_name; extern int remote_option_cnt; -@@ -141,6 +145,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, +@@ -141,6 +144,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, logfile_close(); } @@ -891,47 +974,101 @@ diff --git a/pipe.c b/pipe.c if (remote_option_cnt) { int rc = remote_option_cnt + 1; const char **rv = remote_options; -@@ -148,6 +155,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, +@@ -148,6 +154,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, option_error(); exit_cleanup(RERR_SYNTAX); } -+ if (db_config && (always_checksum || protocol_version >= 30)) ++ if (db_config && always_checksum) + db_read_config(FERROR, db_config); } if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 || -diff --git a/receiver.c b/receiver.c ---- a/receiver.c -+++ b/receiver.c -@@ -43,11 +43,13 @@ extern int basis_dir_cnt; - extern int make_backups; - extern int cleanup_got_literal; - extern int remove_source_files; -+extern int always_checksum; - extern int append_mode; - extern int sparse_files; - extern int keep_partial; - extern int checksum_seed; - extern int inplace; -+extern int use_db; - extern int delay_updates; - extern mode_t orig_umask; - extern struct stats stats; -@@ -399,6 +401,9 @@ int recv_files(int f_in, char *local_name) - if (verbose > 2) - rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used); +diff --git a/rsync.yo b/rsync.yo +--- a/rsync.yo ++++ b/rsync.yo +@@ -317,6 +317,7 @@ to the detailed description below for a complete description. verb( + -q, --quiet suppress non-error messages + --no-motd suppress daemon-mode MOTD (see caveat) + -c, --checksum skip based on checksum, not mod-time & size ++ --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums + -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) + --no-OPTION turn off an implied OPTION (e.g. --no-D) + -r, --recursive recurse into directories +@@ -533,6 +534,47 @@ checksum that is generated as the file is transferred, but that + automatic after-the-transfer verification has nothing to do with this + option's before-the-transfer "Does this file need to be updated?" check. -+ if (use_db && !always_checksum) -+ db_connect(); -+ - if (delay_updates) - delayed_bits = bitbag_create(cur_flist->used + 1); ++dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read ++that holds connection details for a database of checksum information. ++When combined with the bf(--checksum) (bf(-c)) option, rsync will try to ++use cached checksum information from the DB, and will update it if it is ++missing. ++ ++The currently supported DB choices are MySQL and SQLite. For example, a ++MySQL configuration might look like this: ++ ++verb( dbtype: mysql ++ dbhost: 127.0.0.1 ++ dbname: rsyncdb ++ dbuser: rsyncuser ++ dbpass: somepass ++ port: 3306 ++ thishost: hostname ) ++ ++And a SQLite configuration might look like this: ++ ++verb( dbtype: SQLite ++ dbname: /var/cache/rsync/sum.db ) ++ ++This option only affects one side of a transfer. See the ++bf(--remote-option) option for a way to specify the option for both ++sides of the transfer (with each side reading the config file from ++their local filesystem). For example: ++ ++verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ ) ++ ++See the perl script "rsyncdb" in the support directory of the source code ++(which may also be installed in /usr/bin) for a way to create the tables, ++populate the mounted-disk information, check files against their checksums, ++and update both the MD4 and MD5 checksums for files at the same time (since ++an rsync copy will only update one or the other). ++ ++You can use a single MySQL DB for all your hosts if you give each one ++their own "thishost" name and setup their device-mapping data. Or feel ++free to use separate databases, separate servers, etc. See the rsync ++daemon's "db config" parameter for how to configure a daemon to use a DB ++(since a client cannot control this parameter on a daemon). ++ + dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick + way of saying you want recursion and want to preserve almost + everything (with -H being a notable omission). +diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo +--- a/rsyncd.conf.yo ++++ b/rsyncd.conf.yo +@@ -270,6 +270,18 @@ is daemon. This setting has no effect if the "log file" setting is a + non-empty string (either set in the per-modules settings, or inherited + from the global settings). -diff --git a/support/dbupdate b/support/dbupdate ++dit(bf(db config)) This parameter specifies a config file to read that ++holds connection details for a database of checksum information. ++ ++The config file will be read-in prior to any chroot restrictions, but ++the connection occurs from inside the chroot. This means that you ++should use a socket connection (e.g. 127.0.0.1 rather than localhost) ++for a MySQL config from inside a chroot. For SQLite, the DB file must ++be placed inside the chroot (though it can be placed outside the ++transfer dir if you configured an inside-chroot path). ++ ++See the bf(--db=CONFIG_FILE) option for full details. ++ + dit(bf(max verbosity)) The "max verbosity" option allows you to control + the maximum amount of verbose information that you'll allow the daemon to + generate (since the information goes into the log file). The default is 1, +diff --git a/support/rsyncdb b/support/rsyncdb new file mode 100755 --- /dev/null -+++ b/support/dbupdate -@@ -0,0 +1,281 @@ ++++ b/support/rsyncdb +@@ -0,0 +1,331 @@ +#!/usr/bin/perl -w +use strict; + @@ -946,6 +1083,7 @@ new file mode 100755 +&Getopt::Long::Configure('bundling'); +&usage if !&GetOptions( + 'db=s' => \( my $db_config ), ++ 'init' => \( my $init_db ), + 'mounts|m' => \( my $update_mounts ), + 'recurse|r' => \( my $recurse_opt ), + 'check|c' => \( my $check_opt ), @@ -967,10 +1105,13 @@ new file mode 100755 +die "You must define at least dbtype and dbname in $db_config\n" + unless defined $config{'dbtype'} && defined $config{'dbname'}; + ++my $sqlite = $config{'dbtype'} =~ /^sqlite$/i; ++ +my $thishost = $config{'thishost'} || 'localhost'; + -+my $connect = 'DBI:' . $config{'dbtype'} . ':database=' . $config{'dbname'}; -+$connect =~ s/:database=/:dbname=/ if $config{'dbtype'} eq 'SQLite'; ++my $connect = 'DBI:' . $config{'dbtype'} . ':'; ++$connect .= 'dbname=' . $config{'dbname'} if $sqlite; ++$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db; +$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'}; +$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'}; + @@ -981,6 +1122,45 @@ new file mode 100755 + $dbh->disconnect if defined $dbh; +} + ++if ($init_db) { ++ my $unsigned = $sqlite ? '' : 'unsigned'; ++ my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT'; ++ my $dbname = $config{'dbname'}; ++ ++ if (!$sqlite) { ++ $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`"); ++ $dbh->do("USE `$dbname`"); ++ } ++ ++ print "Dropping old tables (if they exist) ...\n" if $verbosity; ++ $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr; ++ $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr; ++ ++ print "Creating empty tables ...\n" if $verbosity; ++ $dbh->do(" ++ CREATE TABLE disk ( ++ disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment, ++ devno bigint $unsigned NOT NULL, ++ host varchar(256) NOT NULL default 'localhost', ++ mounted tinyint NOT NULL default '1', ++ comment varchar(256) default NULL ++ )") or die $dbh->errstr; ++ ++ $dbh->do(" ++ CREATE TABLE inode_map ( ++ disk_id integer $unsigned NOT NULL, ++ ino bigint $unsigned NOT NULL, ++ size bigint $unsigned NOT NULL, ++ mtime bigint NOT NULL, ++ ctime bigint NOT NULL, ++ sum_type tinyint NOT NULL default '0', ++ checksum binary(16) NOT NULL, ++ PRIMARY KEY (disk_id,ino,sum_type) ++ )") or die $dbh->errstr; ++ ++ exit unless $update_mounts; ++} ++ +my $sel_disk_H = $dbh->prepare(" + SELECT disk_id, devno, mounted, comment + FROM disk @@ -999,7 +1179,7 @@ new file mode 100755 + WHERE disk_id = ? + ") or die $dbh->errstr; + -+my $row_id = $config{'dbtype'} eq 'SQLite' ? 'ROWID' : 'ID'; ++my $row_id = $sqlite ? 'ROWID' : 'ID'; +my $sel_lastid_H = $dbh->prepare(" + SELECT LAST_INSERT_$row_id() + ") or die $dbh->errstr; @@ -1039,15 +1219,18 @@ new file mode 100755 + if (defined $mounts{$devno}) { + if ($comment ne $mounts{$devno}) { + if ($mounted) { ++ print "Umounting $comment ($thishost:$devno)\n" if $verbosity; + $up_disk_H->execute(0, $disk_id); + } + next; + } + if (!$mounted) { ++ print "Mounting $comment ($thishost:$devno)\n" if $verbosity; + $up_disk_H->execute(1, $disk_id); + } + } else { + if ($mounted) { ++ print "Umounting $comment ($thishost:$devno)\n" if $verbosity; + $up_disk_H->execute(0, $disk_id); + } + next; @@ -1062,11 +1245,13 @@ new file mode 100755 +if ($update_mounts) { + while (my($devno, $comment) = each %mounts) { + next if $disk_id{$devno}; ++ print "Adding $comment ($thishost:$devno)\n" if $verbosity; + $ins_disk_H->execute($devno, $thishost, 1, $comment); + $sel_lastid_H->execute; + ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array; + $sel_lastid_H->finish; + } ++ exit; +} + +my $start_dir = cwd(); @@ -1206,7 +1391,9 @@ new file mode 100755 + +Options: + --db=FILE Specify the config FILE to read for the DB info. -+ -m, --mounts Update mount info. ++ --init Create (recreate) needed tables (making them empty). ++ No DIR scanning, but can be combined with --mounts. ++ -m, --mounts Update mount info. Does no DIR scanning. + -r, --recurse Scan files in subdirectories too. + -c, --check Check if the checksums are right (doesn't update). + -v, --verbose Mention what we're doing. Repeat for more info. diff --git a/detect-renamed.diff b/detect-renamed.diff index e28ed1d..9bff407 100644 --- a/detect-renamed.diff +++ b/detect-renamed.diff @@ -117,7 +117,7 @@ diff --git a/flist.c b/flist.c static void send_directory(int f, struct file_list *flist, char *fbuf, int len, int flags); -@@ -2259,6 +2301,25 @@ struct file_list *recv_file_list(int f) +@@ -2263,6 +2305,25 @@ struct file_list *recv_file_list(int f) flist_sort_and_clean(flist, relative_paths); diff --git a/fileflags.diff b/fileflags.diff index ca7c1aa..86c0a4e 100644 --- a/fileflags.diff +++ b/fileflags.diff @@ -85,7 +85,7 @@ diff --git a/flist.c b/flist.c extern int uid_ndx; extern int gid_ndx; extern int eol_nulls; -@@ -386,6 +387,9 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -390,6 +391,9 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, { static time_t modtime; static mode_t mode; @@ -95,7 +95,7 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS static int64 dev; #endif -@@ -415,6 +419,14 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -419,6 +423,14 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, xflags |= XMIT_SAME_MODE; else mode = file->mode; @@ -110,7 +110,7 @@ diff --git a/flist.c b/flist.c if ((preserve_devices && IS_DEVICE(mode)) || (preserve_specials && IS_SPECIAL(mode))) { -@@ -529,6 +541,10 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -533,6 +545,10 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, } if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); @@ -121,7 +121,7 @@ diff --git a/flist.c b/flist.c if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) write_int(f, uid); -@@ -617,6 +633,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -621,6 +637,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, { static int64 modtime; static mode_t mode; @@ -131,7 +131,7 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS static int64 dev; #endif -@@ -752,6 +771,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -756,6 +775,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, if (chmod_modes && !S_ISLNK(mode)) mode = tweak_mode(mode, chmod_modes); @@ -142,7 +142,7 @@ diff --git a/flist.c b/flist.c if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) -@@ -872,6 +895,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -876,6 +899,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); } file->mode = mode; @@ -153,7 +153,7 @@ diff --git a/flist.c b/flist.c if (preserve_uid) F_OWNER(file) = uid; if (preserve_gid) { -@@ -1222,6 +1249,10 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1226,6 +1253,10 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, OPT_EXTRA(file, 0)->unum = (uint32)(st.st_size >> 32); } file->mode = st.st_mode; diff --git a/ignore-case.diff b/ignore-case.diff index 169bf91..e7a284f 100644 --- a/ignore-case.diff +++ b/ignore-case.diff @@ -55,7 +55,7 @@ diff --git a/flist.c b/flist.c extern int ignore_errors; extern int numeric_ids; extern int recurse; -@@ -2696,6 +2697,7 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) +@@ -2700,6 +2701,7 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) { int dif; const uchar *c1, *c2; @@ -63,7 +63,7 @@ diff --git a/flist.c b/flist.c enum fnc_state state1, state2; enum fnc_type type1, type2; enum fnc_type t_path = protocol_version >= 29 ? t_PATH : t_ITEM; -@@ -2806,7 +2808,15 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) +@@ -2810,7 +2812,15 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) if (type1 != type2) return type1 == t_PATH ? 1 : -1; } diff --git a/link-by-hash.diff b/link-by-hash.diff index 96bea34..014b23e 100644 --- a/link-by-hash.diff +++ b/link-by-hash.diff @@ -35,7 +35,7 @@ diff --git a/flist.c b/flist.c extern char curr_dir[MAXPATHLEN]; -@@ -830,7 +831,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -834,7 +835,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, extra_len += EXTRA_LEN; #endif diff --git a/slow-down.diff b/slow-down.diff index 1968278..5aa0c39 100644 --- a/slow-down.diff +++ b/slow-down.diff @@ -25,7 +25,7 @@ diff --git a/flist.c b/flist.c extern struct stats stats; extern char *filesfrom_host; -@@ -1553,6 +1554,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, +@@ -1557,6 +1558,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, } send_file_name(f, flist, fbuf, NULL, flags, filter_level); diff --git a/transliterate.diff b/transliterate.diff index 69ae5d1..3d2a803 100644 --- a/transliterate.diff +++ b/transliterate.diff @@ -26,7 +26,7 @@ diff --git a/flist.c b/flist.c #define PTR_SIZE (sizeof (struct file_struct *)) int io_error; -@@ -612,6 +615,24 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, +@@ -616,6 +619,24 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, stats.total_size += F_LENGTH(file); } @@ -51,7 +51,7 @@ diff --git a/flist.c b/flist.c static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { -@@ -680,6 +701,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -684,6 +705,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } #endif diff --git a/usermap.diff b/usermap.diff index f09797a..123cd9d 100644 --- a/usermap.diff +++ b/usermap.diff @@ -18,7 +18,7 @@ diff --git a/flist.c b/flist.c extern char curr_dir[MAXPATHLEN]; -@@ -760,7 +761,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -764,7 +765,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, uid = (uid_t)read_varint(f); if (xflags & XMIT_USER_NAME_FOLLOWS) uid = recv_user_name(f, uid); @@ -27,7 +27,7 @@ diff --git a/flist.c b/flist.c uid = match_uid(uid); } } -@@ -772,7 +773,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -776,7 +777,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, gid_flags = 0; if (xflags & XMIT_GROUP_NAME_FOLLOWS) gid = recv_group_name(f, gid, &gid_flags); @@ -36,7 +36,7 @@ diff --git a/flist.c b/flist.c gid = match_gid(gid, &gid_flags); } } -@@ -2166,8 +2167,13 @@ struct file_list *recv_file_list(int f) +@@ -2170,8 +2171,13 @@ struct file_list *recv_file_list(int f) int dstart, flags; int64 start_read; -- 2.34.1