The patches for 3.0.1pre3.

[rsync/rsync-patches.git] / db.diff

Added some DB-access routines to help rsync keep extra filesystem info
about the files it is dealing with.  This adds both the --db=CONFIG_FILE
option and the "db config" daemon parameter.

For the moment this only adds checksum caching when the --checksum option
is used.  Future improvments may include:

 - Updating of MD5 checksums when transferring any file, even w/o -c.

 - Caching of path info that allows for the finding of files to use for
   moving/linking/copying/alternate-basis-use.

 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).

To use this patch, run these commands for a successful build:

    patch -p1 <patches/remote-option.diff
    patch -p1 <patches/db.diff
    ./configure                               (optional if already run)
    make

diff --git a/Makefile.in b/Makefile.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -35,7 +35,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
 	util.o main.o checksum.o match.o syscall.o log.o backup.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
-	fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
+	fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
 OBJS3=progress.o pipe.o
 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
 popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
diff --git a/checksum.c b/checksum.c
--- a/checksum.c
+++ b/checksum.c
@@ -23,6 +23,7 @@
 
 extern int checksum_seed;
 extern int protocol_version;
+extern int use_db;
 
 int csum_length = SHORT_SUM_LENGTH; /* initial value */
 
@@ -100,10 +101,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
 	}
 }
 
-void file_checksum(char *fname, char *sum, OFF_T size)
+void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
 {
 	struct map_struct *buf;
-	OFF_T i, len = size;
+	OFF_T i, len = st_p->st_size;
 	md_context m;
 	int32 remainder;
 	int fd;
@@ -114,7 +115,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
 	if (fd == -1)
 		return;
 
-	buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
+	buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
 
 	if (protocol_version >= 30) {
 		md5_begin(&m);
@@ -148,6 +149,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
 		mdfour_result(&m, (uchar *)sum);
 	}
 
+	if (use_db)
+		db_set_checksum(fname, st_p, sum);
+
 	close(fd);
 	unmap_file(buf);
 }
diff --git a/cleanup.c b/cleanup.c
--- a/cleanup.c
+++ b/cleanup.c
@@ -27,6 +27,7 @@ extern int am_daemon;
 extern int io_error;
 extern int keep_partial;
 extern int got_xfer_error;
+extern int use_db;
 extern char *partial_dir;
 extern char *logfile_name;
 
@@ -124,6 +125,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 		/* FALLTHROUGH */
 #include "case_N.h"
 
+		if (use_db)
+			db_disconnect();
+
+		/* FALLTHROUGH */
+#include "case_N.h"
+
 		if (cleanup_child_pid != -1) {
 			int status;
 			int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
diff --git a/clientserver.c b/clientserver.c
--- a/clientserver.c
+++ b/clientserver.c
@@ -42,6 +42,7 @@ extern int numeric_ids;
 extern int filesfrom_fd;
 extern int remote_protocol;
 extern int protocol_version;
+extern int always_checksum;
 extern int io_timeout;
 extern int no_detach;
 extern int write_batch;
@@ -49,6 +50,7 @@ extern int default_af_hint;
 extern int logfile_format_has_i;
 extern int logfile_format_has_o_or_i;
 extern mode_t orig_umask;
+extern char *db_config;
 extern char *bind_address;
 extern char *sockopts;
 extern char *config_file;
@@ -782,6 +784,12 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 	} else if (am_root < 0) /* Treat --fake-super from client as --super. */
 		am_root = 2;
 
+	db_config = lp_db_config(i);
+	if (!*db_config || (!always_checksum && protocol_version < 30))
+		db_config = NULL;
+	else
+		db_read_config(FLOG, db_config);
+
 	if (filesfrom_fd == 0)
 		filesfrom_fd = f_in;
 
diff --git a/configure.in b/configure.in
--- a/configure.in
+++ b/configure.in
@@ -969,6 +969,8 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
     fi
 fi
 
+LIBS="$LIBS -lmysqlclient -lsqlite3"
+
 case "$CC" in
 ' checker'*|checker*)
     AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
diff --git a/db.c b/db.c
new file mode 100644
--- /dev/null
+++ b/db.c
@@ -0,0 +1,557 @@
+/*
+ * Routines to access extended file info via DB.
+ *
+ * Copyright (C) 2008 Wayne Davison
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, visit the http://fsf.org website.
+ */
+
+#include "rsync.h"
+#include "ifuncs.h"
+
+#define USE_MYSQL
+#define USE_SQLITE
+
+#ifdef USE_MYSQL
+#include <mysql/mysql.h>
+#include <mysql/errmsg.h>
+#endif
+#ifdef USE_SQLITE
+#include <sqlite3.h>
+#endif
+
+extern int protocol_version;
+extern int checksum_len;
+
+#define DB_TYPE_NONE 0
+#define DB_TYPE_MYSQL 1
+#define DB_TYPE_SQLITE 2
+
+int use_db = DB_TYPE_NONE;
+
+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
+static unsigned int dbport = 0;
+
+static union {
+#ifdef USE_MYSQL
+    MYSQL *mysql;
+#endif
+#ifdef USE_SQLITE
+    sqlite3 *sqlite;
+#endif
+    void *all;
+} dbh;
+
+#define SEL_DEV 0
+#define SEL_SUM 1
+#define REP_SUM 2
+#define MAX_PREP_CNT 3
+
+static union {
+#ifdef USE_MYSQL
+    MYSQL_STMT *mysql;
+#endif
+#ifdef USE_SQLITE
+    sqlite3_stmt *sqlite;
+#endif
+    void *all;
+} statements[MAX_PREP_CNT];
+
+static int md_num;
+static enum logcode log_code;
+
+static unsigned int bind_disk_id;
+static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
+static char bind_thishost[256], bind_sum[MAX_DIGEST_LEN];
+static int bind_thishost_len;
+
+static unsigned int prior_disk_id = 0;
+static unsigned long long prior_devno = 0;
+
+int db_read_config(enum logcode code, const char *config_file)
+{
+	char buf[2048], *cp;
+	FILE *fp;
+	int lineno = 0;
+
+	log_code = code;
+
+	bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
+
+	if (!(fp = fopen(config_file, "r"))) {
+		rsyserr(log_code, errno, "unable to open %s", config_file);
+		return 0;
+	}
+	while (fgets(buf, sizeof buf, fp)) {
+		lineno++;
+		if ((cp = strchr(buf, '#')) == NULL
+		 && (cp = strchr(buf, '\r')) == NULL
+		 && (cp = strchr(buf, '\n')) == NULL)
+			cp = buf + strlen(buf);
+		while (cp != buf && isSpace(cp-1)) cp--;
+		*cp = '\0';
+
+		if (!*buf)
+			continue;
+
+		if (!(cp = strchr(buf, ':')))
+			goto invalid_line;
+		*cp++ = '\0';
+
+		while (isSpace(cp)) cp++;
+		if (strcasecmp(buf, "dbhost") == 0)
+			dbhost = strdup(cp);
+		else if (strcasecmp(buf, "dbuser") == 0)
+			dbuser = strdup(cp);
+		else if (strcasecmp(buf, "dbpass") == 0)
+			dbpass = strdup(cp);
+		else if (strcasecmp(buf, "dbname") == 0)
+			dbname = strdup(cp);
+		else if (strcasecmp(buf, "dbport") == 0)
+			dbport = atoi(cp);
+		else if (strcasecmp(buf, "thishost") == 0)
+			bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
+		else if (strcasecmp(buf, "dbtype") == 0) {
+#ifdef USE_MYSQL
+			if (strcasecmp(cp, "mysql") == 0) {
+				use_db = DB_TYPE_MYSQL;
+				continue;
+			}
+#endif
+#ifdef USE_SQLITE
+			if (strcasecmp(cp, "sqlite") == 0) {
+				use_db = DB_TYPE_SQLITE;
+				continue;
+			}
+#endif
+			rprintf(log_code,
+			    "Unsupported dbtype on line #%d in %s.\n",
+			    lineno, config_file);
+			use_db = DB_TYPE_NONE;
+			return 0;
+		} else {
+		  invalid_line:
+			rprintf(log_code, "Invalid line #%d in %s\n",
+				lineno, config_file);
+			use_db = DB_TYPE_NONE;
+			return 0;
+		}
+	}
+	fclose(fp);
+
+	if (bind_thishost_len >= (int)sizeof bind_thishost)
+		bind_thishost_len = sizeof bind_thishost - 1;
+
+	if (!use_db || !dbname) {
+		rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
+		use_db = DB_TYPE_NONE;
+		return 0;
+	}
+
+	md_num = protocol_version >= 30 ? 5 : 4;
+
+	return 1;
+}
+
+#ifdef USE_MYSQL
+static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
+{
+	va_list ap;
+	char *query;
+	int qlen, param_cnt;
+	MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
+
+	if (stmt == NULL)
+		out_of_memory("prepare_mysql");
+
+	va_start(ap, fmt);
+	qlen = vasprintf(&query, fmt, ap);
+	va_end(ap);
+	if (qlen < 0)
+		out_of_memory("prepare_mysql");
+
+	if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
+		rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
+		return NULL;
+	}
+	free(query);
+
+	if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
+		rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
+			param_cnt, bind_cnt);
+		return NULL;
+	}
+	if (bind_cnt)
+		mysql_stmt_bind_param(stmt, binds);
+
+	return stmt;
+}
+#endif
+
+#ifdef USE_MYSQL
+static int db_connect_mysql(void)
+{
+	MYSQL_BIND binds[10];
+
+	if (!(dbh.mysql = mysql_init(NULL)))
+		out_of_memory("db_read_config");
+
+	if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
+		return 0;
+
+	memset(binds, 0, sizeof binds);
+	binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[0].buffer = &bind_devno;
+	binds[1].buffer_type = MYSQL_TYPE_STRING;
+	binds[1].buffer = &bind_thishost;
+	binds[1].buffer_length = bind_thishost_len;
+	statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
+		"SELECT disk_id"
+		" FROM disk"
+		" WHERE devno = ? AND host = ? AND mounted = 1");
+	if (!statements[SEL_DEV].mysql)
+		return 0;
+
+	memset(binds, 0, sizeof binds);
+	binds[0].buffer_type = MYSQL_TYPE_LONG;
+	binds[0].buffer = &bind_disk_id;
+	binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[1].buffer = &bind_ino;
+	binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[2].buffer = &bind_size;
+	binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[3].buffer = &bind_mtime;
+	binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[4].buffer = &bind_ctime;
+	statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
+		"SELECT checksum"
+		" FROM inode_map"
+		" WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+		"   AND size = ? AND mtime = ? AND ctime = ?",
+		md_num);
+	if (!statements[SEL_SUM].mysql)
+		return 0;
+
+	memset(binds, 0, sizeof binds);
+	binds[0].buffer_type = MYSQL_TYPE_LONG;
+	binds[0].buffer = &bind_disk_id;
+	binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[1].buffer = &bind_ino;
+	binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[2].buffer = binds[6].buffer = &bind_size;
+	binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[3].buffer = binds[7].buffer = &bind_mtime;
+	binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
+	binds[4].buffer = binds[8].buffer = &bind_ctime;
+	binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
+	binds[5].buffer = binds[9].buffer = &bind_sum;
+	binds[5].buffer_length = binds[9].buffer_length = checksum_len;
+	statements[REP_SUM].mysql = prepare_mysql(binds, 10,
+		"INSERT INTO inode_map"
+		" SET disk_id = ?, ino = ?, sum_type = %d,"
+		"     size = ?, mtime = ?, ctime = ?, checksum = ?"
+		" ON DUPLICATE KEY"
+		" UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
+		md_num, md_num);
+	if (!statements[REP_SUM].mysql)
+		return 0;
+
+	return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int db_connect_sqlite(void)
+{
+	char *sql;
+
+#if 0
+	if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
+		return 0;
+#else
+	if (sqlite3_open(dbname, &dbh.sqlite) != 0)
+		return 0;
+#endif
+
+	sql = "SELECT disk_id"
+	    " FROM disk"
+	    " WHERE devno = ? AND host = ? AND mounted = 1";
+	if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
+		return 0;
+
+	if (asprintf(&sql,
+	    "SELECT checksum"
+	    " FROM inode_map"
+	    " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+	    "   AND size = ? AND mtime = ? AND ctime = ?",
+	    md_num) < 0
+	 || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
+		return 0;
+	free(sql);
+
+	if (asprintf(&sql,
+	    "INSERT OR REPLACE INTO inode_map"
+	    " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
+	    " VALUES(?, ?, %d, ?, ?, ?, ?)",
+	    md_num) < 0
+	 || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
+		return 0;
+	free(sql);
+
+	return 1;
+}
+#endif
+
+int db_connect(void)
+{
+	switch (use_db) {
+#ifdef USE_MYSQL
+	case DB_TYPE_MYSQL:
+		if (db_connect_mysql())
+			return 1;
+		break;
+#endif
+#ifdef USE_SQLITE
+	case DB_TYPE_SQLITE:
+		if (db_connect_sqlite())
+			return 1;
+		break;
+#endif
+	}
+
+	rprintf(log_code, "Unable to connect to DB\n");
+	db_disconnect();
+	use_db = DB_TYPE_NONE;
+
+	return 0;
+}
+
+void db_disconnect(void)
+{
+	int ndx;
+
+	if (!dbh.all)
+		return;
+
+	for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
+		if (statements[ndx].all) {
+			switch (use_db) {
+#ifdef USE_MYSQL
+			case DB_TYPE_MYSQL:
+				mysql_stmt_close(statements[ndx].mysql);
+				break;
+#endif
+#ifdef USE_SQLITE
+			case DB_TYPE_SQLITE:
+				sqlite3_finalize(statements[ndx].sqlite);
+				break;
+#endif
+			}
+			statements[ndx].all = NULL;
+		}
+	}
+
+	switch (use_db) {
+#ifdef USE_MYSQL
+	case DB_TYPE_MYSQL:
+		mysql_close(dbh.mysql);
+		break;
+#endif
+#ifdef USE_SQLITE
+	case DB_TYPE_SQLITE:
+		sqlite3_close(dbh.sqlite);
+		break;
+#endif
+	}
+
+	dbh.all = NULL;
+}
+
+static MYSQL_STMT *exec_mysql(int ndx)
+{
+	MYSQL_STMT *stmt = statements[ndx].mysql;
+	int rc;
+
+	if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
+		db_disconnect();
+		if (db_connect()) {
+			stmt = statements[ndx].mysql;
+			rc = mysql_stmt_execute(stmt);
+		}
+	}
+	if (rc != 0) {
+		rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
+		return NULL;
+	}
+
+	return stmt;
+}
+
+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
+{
+	unsigned long length[32];
+	my_bool is_null[32], error[32];
+	MYSQL_STMT *stmt;
+	int i, rc;
+
+	if (bind_cnt > 32)
+		exit_cleanup(RERR_UNSUPPORTED);
+
+	if ((stmt = exec_mysql(ndx)) == NULL)
+		return 0;
+
+	for (i = 0; i < bind_cnt; i++) {
+		binds[i].is_null = &is_null[i];
+		binds[i].length = &length[i];
+		binds[i].error = &error[i];
+	}
+	mysql_stmt_bind_result(stmt, binds);
+
+	if ((rc = mysql_stmt_fetch(stmt)) != 0) {
+		if (rc != MYSQL_NO_DATA) {
+			rprintf(log_code, "SELECT fetch failed: %s\n",
+				mysql_stmt_error(stmt));
+		}
+		mysql_stmt_free_result(stmt);
+		return 0;
+	}
+
+	mysql_stmt_free_result(stmt);
+
+	return is_null[0] ? 0 : 1;
+}
+
+static void get_disk_id(unsigned long long devno)
+{
+	switch (use_db) {
+#ifdef USE_MYSQL
+	case DB_TYPE_MYSQL: {
+		MYSQL_BIND binds[1];
+
+		bind_devno = devno; /* The one variable SEL_DEV input value. */
+
+		/* Bind where to put the output. */
+		binds[0].buffer_type = MYSQL_TYPE_LONG;
+		binds[0].buffer = &prior_disk_id;
+		if (!fetch_mysql(binds, 1, SEL_DEV))
+			prior_disk_id = 0;
+		break;
+	    }
+#endif
+#ifdef USE_SQLITE
+	case DB_TYPE_SQLITE: {
+		sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
+		sqlite3_bind_int64(stmt, 1, devno);
+		sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+		if (sqlite3_step(stmt) == SQLITE_ROW)
+			prior_disk_id = sqlite3_column_int(stmt, 0);
+		else
+			prior_disk_id = 0;
+		sqlite3_reset(stmt);
+		break;
+	    }
+#endif
+	}
+
+	prior_devno = devno;
+}
+
+int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
+{
+	if (prior_devno != st_p->st_dev)
+		get_disk_id(st_p->st_dev);
+	if (prior_disk_id == 0)
+		return 0;
+
+	switch (use_db) {
+#ifdef USE_MYSQL
+	case DB_TYPE_MYSQL: {
+		MYSQL_BIND binds[1];
+
+		bind_disk_id = prior_disk_id;
+		bind_ino = st_p->st_ino;
+		bind_size = st_p->st_size;
+		bind_mtime = st_p->st_mtime;
+		bind_ctime = st_p->st_ctime;
+
+		binds[0].buffer_type = MYSQL_TYPE_BLOB;
+		binds[0].buffer = sum;
+		binds[0].buffer_length = checksum_len;
+		return fetch_mysql(binds, 1, SEL_SUM);
+	    }
+#endif
+#ifdef USE_SQLITE
+	case DB_TYPE_SQLITE: {
+		sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
+		sqlite3_bind_int(stmt, 1, prior_disk_id);
+		sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+		sqlite3_bind_int64(stmt, 3, st_p->st_size);
+		sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
+		sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			int len = sqlite3_column_bytes(stmt, 0);
+			if (len > MAX_DIGEST_LEN)
+				len = MAX_DIGEST_LEN;
+			memcpy(sum, sqlite3_column_blob(stmt, 0), len);
+			sqlite3_reset(stmt);
+			return 1;
+		}
+		sqlite3_reset(stmt);
+		return 0;
+	    }
+#endif
+	}
+
+	return 0;
+}
+
+int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
+{
+	if (prior_devno != st_p->st_dev)
+		get_disk_id(st_p->st_dev);
+	if (prior_disk_id == 0)
+		return 0;
+
+	switch (use_db) {
+#ifdef USE_MYSQL
+	case DB_TYPE_MYSQL: {
+		bind_disk_id = prior_disk_id;
+		bind_ino = st_p->st_ino;
+		bind_size = st_p->st_size;
+		bind_mtime = st_p->st_mtime;
+		bind_ctime = st_p->st_ctime;
+		memcpy(bind_sum, sum, checksum_len);
+
+		return exec_mysql(REP_SUM) != NULL;
+	    }
+#endif
+#ifdef USE_SQLITE
+	case DB_TYPE_SQLITE: {
+		int rc;
+		sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
+		sqlite3_bind_int(stmt, 1, prior_disk_id);
+		sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+		sqlite3_bind_int64(stmt, 3, st_p->st_size);
+		sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
+		sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
+		sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
+		rc = sqlite3_step(stmt);
+		sqlite3_reset(stmt);
+		return rc == SQLITE_DONE;
+	    }
+#endif
+	}
+
+	return 0;
+}
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -54,6 +54,7 @@ extern int preserve_devices;
 extern int preserve_specials;
 extern int uid_ndx;
 extern int gid_ndx;
+extern int use_db;
 extern int eol_nulls;
 extern int relative_paths;
 extern int implied_dirs;
@@ -1235,14 +1236,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 		memcpy(bp + basename_len, linkname, linkname_len);
 #endif
 
-	if (always_checksum && am_sender && S_ISREG(st.st_mode))
-		file_checksum(thisname, tmp_sum, st.st_size);
-
 	if (am_sender)
 		F_PATHNAME(file) = pathname;
 	else if (!pool)
 		F_DEPTH(file) = extra_len / EXTRA_LEN;
 
+	if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
+		if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
+			file_checksum(thisname, &st, tmp_sum);
+	}
+
 	/* This code is only used by the receiver when it is building
 	 * a list of files for a delete pass. */
 	if (keep_dirlinks && linkname_len && flist) {
@@ -1858,6 +1861,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 		     | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
 	int implied_dot_dir = 0;
 
+	if (use_db)
+		db_connect();
+
 	rprintf(FLOG, "building file list\n");
 	if (show_filelist_p())
 		start_filelist_progress("building file list");
diff --git a/generator.c b/generator.c
--- a/generator.c
+++ b/generator.c
@@ -58,6 +58,7 @@ extern int update_only;
 extern int ignore_existing;
 extern int ignore_non_existing;
 extern int inplace;
+extern int use_db;
 extern int append_mode;
 extern int make_backups;
 extern int csum_length;
@@ -718,7 +719,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
 	   of the file time to determine whether to sync */
 	if (always_checksum > 0 && S_ISREG(st->st_mode)) {
 		char sum[MAX_DIGEST_LEN];
-		file_checksum(fn, sum, st->st_size);
+		if (!use_db || !db_get_checksum(fn, st, sum))
+			file_checksum(fn, st, sum);
 		return memcmp(sum, F_SUM(file), checksum_len) == 0;
 	}
 
@@ -2161,6 +2163,9 @@ void generate_files(int f_out, const char *local_name)
 			: "enabled");
 	}
 
+	if (use_db && always_checksum)
+		db_connect();
+
 	/* Since we often fill up the outgoing socket and then just sit around
 	 * waiting for the other 2 processes to do their thing, we don't want
 	 * to exit on a timeout.  If the data stops flowing, the receiver will
diff --git a/loadparm.c b/loadparm.c
--- a/loadparm.c
+++ b/loadparm.c
@@ -126,6 +126,7 @@ typedef struct
 	char *auth_users;
 	char *charset;
 	char *comment;
+	char *db_config;
 	char *dont_compress;
 	char *exclude;
 	char *exclude_from;
@@ -177,6 +178,7 @@ static service sDefault =
  /* auth_users; */		NULL,
  /* charset; */ 		NULL,
  /* comment; */ 		NULL,
+ /* db_config; */		NULL,
  /* dont_compress; */		DEFAULT_DONT_COMPRESS,
  /* exclude; */			NULL,
  /* exclude_from; */		NULL,
@@ -307,6 +309,7 @@ static struct parm_struct parm_table[] =
  {"auth users",        P_STRING, P_LOCAL, &sDefault.auth_users,        NULL,0},
  {"charset",           P_STRING, P_LOCAL, &sDefault.charset,           NULL,0},
  {"comment",           P_STRING, P_LOCAL, &sDefault.comment,           NULL,0},
+ {"db config",         P_STRING, P_LOCAL, &sDefault.db_config,         NULL,0},
  {"dont compress",     P_STRING, P_LOCAL, &sDefault.dont_compress,     NULL,0},
  {"exclude from",      P_STRING, P_LOCAL, &sDefault.exclude_from,      NULL,0},
  {"exclude",           P_STRING, P_LOCAL, &sDefault.exclude,           NULL,0},
@@ -400,6 +403,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Globals.rsync_port)
 FN_LOCAL_STRING(lp_auth_users, auth_users)
 FN_LOCAL_STRING(lp_charset, charset)
 FN_LOCAL_STRING(lp_comment, comment)
+FN_LOCAL_STRING(lp_db_config, db_config)
 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
 FN_LOCAL_STRING(lp_exclude, exclude)
 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
diff --git a/main.c b/main.c
--- a/main.c
+++ b/main.c
@@ -49,6 +49,7 @@ extern int copy_unsafe_links;
 extern int keep_dirlinks;
 extern int preserve_hard_links;
 extern int protocol_version;
+extern int always_checksum;
 extern int file_total;
 extern int recurse;
 extern int xfer_dirs;
@@ -73,6 +74,7 @@ extern char *partial_dir;
 extern char *dest_option;
 extern char *basis_dir[];
 extern char *rsync_path;
+extern char *db_config;
 extern char *shell_cmd;
 extern char *batch_name;
 extern char *password_file;
@@ -1482,6 +1484,9 @@ int main(int argc,char *argv[])
 		exit_cleanup(RERR_SYNTAX);
 	}
 
+	if (db_config && (always_checksum || protocol_version >= 30))
+		db_read_config(FERROR, db_config);
+
 	if (am_server) {
 		set_nonblocking(STDIN_FILENO);
 		set_nonblocking(STDOUT_FILENO);
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -92,6 +92,7 @@ int use_qsort = 0;
 char *files_from = NULL;
 int filesfrom_fd = -1;
 char *filesfrom_host = NULL;
+char *db_config = NULL;
 int eol_nulls = 0;
 int protect_args = 0;
 int human_readable = 0;
@@ -321,6 +322,7 @@ void usage(enum logcode F)
   rprintf(F," -q, --quiet                 suppress non-error messages\n");
   rprintf(F,"     --no-motd               suppress daemon-mode MOTD (see manpage caveat)\n");
   rprintf(F," -c, --checksum              skip based on checksum, not mod-time & size\n");
+  rprintf(F,"     --db=CONFIG_FILE        specify a config file for FS DB\n");
   rprintf(F," -a, --archive               archive mode; equals -rlptgoD (no -H,-A,-X)\n");
   rprintf(F,"     --no-OPTION             turn off an implied OPTION (e.g. --no-D)\n");
   rprintf(F," -r, --recursive             recurse into directories\n");
@@ -579,6 +581,7 @@ static struct poptOption long_options[] = {
   {"checksum",        'c', POPT_ARG_VAL,    &always_checksum, 1, 0, 0 },
   {"no-checksum",      0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
   {"no-c",             0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
+  {"db",               0,  POPT_ARG_STRING, &db_config, 0, 0, 0 },
   {"block-size",      'B', POPT_ARG_LONG,   &block_size, 0, 0, 0 },
   {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
   {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
diff --git a/pipe.c b/pipe.c
--- a/pipe.c
+++ b/pipe.c
@@ -26,6 +26,10 @@ extern int am_sender;
 extern int am_server;
 extern int blocking_io;
 extern int filesfrom_fd;
+extern int always_checksum;
+extern int protocol_version;
+extern int use_db;
+extern char *db_config;
 extern mode_t orig_umask;
 extern char *logfile_name;
 extern int remote_option_cnt;
@@ -141,6 +145,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
 			logfile_close();
 		}
 
+		use_db = 0;
+		db_config = NULL;
+
 		if (remote_option_cnt) {
 			int rc = remote_option_cnt + 1;
 			const char **rv = remote_options;
@@ -148,6 +155,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
 				option_error();
 				exit_cleanup(RERR_SYNTAX);
 			}
+			if (db_config && (always_checksum || protocol_version >= 30))
+				db_read_config(FERROR, db_config);
 		}
 
 		if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
diff --git a/receiver.c b/receiver.c
--- a/receiver.c
+++ b/receiver.c
@@ -43,11 +43,13 @@ extern int basis_dir_cnt;
 extern int make_backups;
 extern int cleanup_got_literal;
 extern int remove_source_files;
+extern int always_checksum;
 extern int append_mode;
 extern int sparse_files;
 extern int keep_partial;
 extern int checksum_seed;
 extern int inplace;
+extern int use_db;
 extern int delay_updates;
 extern mode_t orig_umask;
 extern struct stats stats;
@@ -399,6 +401,9 @@ int recv_files(int f_in, char *local_name)
 	if (verbose > 2)
 		rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used);
 
+	if (use_db && !always_checksum)
+		db_connect();
+
 	if (delay_updates)
 		delayed_bits = bitbag_create(cur_flist->used + 1);
 
diff --git a/support/dbupdate b/support/dbupdate
new file mode 100755
--- /dev/null
+++ b/support/dbupdate
@@ -0,0 +1,281 @@
+#!/usr/bin/perl -w
+use strict;
+
+use DBI;
+use Getopt::Long;
+use Cwd qw(abs_path cwd);
+use Digest::MD4;
+use Digest::MD5;
+
+my $MOUNT_FILE = '/etc/mtab';
+
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+    'db=s' => \( my $db_config ),
+    'mounts|m' => \( my $update_mounts ),
+    'recurse|r' => \( my $recurse_opt ),
+    'check|c' => \( my $check_opt ),
+    'verbose|v+' => \( my $verbosity = 0 ),
+    'help|h' => \( my $help_opt ),
+);
+&usage if $help_opt || !defined $db_config;
+
+my %config;
+open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
+while (<IN>) {
+    s/[#\r\n].*//s;
+    next if /^$/;
+    my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
+    $config{$key} = $val;
+}
+close IN;
+
+die "You must define at least dbtype and dbname in $db_config\n"
+    unless defined $config{'dbtype'} && defined $config{'dbname'};
+
+my $thishost = $config{'thishost'} || 'localhost';
+
+my $connect = 'DBI:' . $config{'dbtype'} . ':database=' . $config{'dbname'};
+$connect =~ s/:database=/:dbname=/ if $config{'dbtype'} eq 'SQLite';
+$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
+$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
+
+my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
+    or die "DB connection failed\n";
+
+END {
+    $dbh->disconnect if defined $dbh;
+}
+
+my $sel_disk_H = $dbh->prepare("
+    SELECT disk_id, devno, mounted, comment
+    FROM disk
+    WHERE host = ?
+    ") or die $dbh->errstr;
+
+my $ins_disk_H = $dbh->prepare("
+    INSERT INTO disk
+    (devno, host, mounted, comment)
+    VALUES(?, ?, ?, ?)
+    ") or die $dbh->errstr;
+
+my $up_disk_H = $dbh->prepare("
+    UPDATE disk
+    SET mounted = ?
+    WHERE disk_id = ?
+    ") or die $dbh->errstr;
+
+my $row_id = $config{'dbtype'} eq 'SQLite' ? 'ROWID' : 'ID';
+my $sel_lastid_H = $dbh->prepare("
+    SELECT LAST_INSERT_$row_id()
+    ") or die $dbh->errstr;
+
+my $sel_sum_H = $dbh->prepare("
+    SELECT sum_type, checksum
+    FROM inode_map
+    WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
+    ") or die $dbh->errstr;
+
+my $rep_sum_H = $dbh->prepare("
+    REPLACE INTO inode_map
+    (disk_id, ino, size, mtime, ctime, sum_type, checksum)
+    VALUES(?, ?, ?, ?, ?, ?, ?)
+    ") or die $dbh->errstr;
+
+my %mounts;
+if ($update_mounts) {
+    open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
+    while (<IN>) {
+	my($devname, $mnt) = (split)[0,1];
+	next unless $devname =~ m#^/dev#;
+	my($devno) = (stat($mnt))[0];
+	if (!defined $devno) {
+	    warn "Unable to stat $mnt: $!\n";
+	    next;
+	}
+	$mounts{$devno} = "$devname on $mnt";
+    }
+    close IN;
+}
+
+my %disk_id;
+$sel_disk_H->execute($thishost);
+while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
+    if ($update_mounts) {
+	if (defined $mounts{$devno}) {
+	    if ($comment ne $mounts{$devno}) {
+		if ($mounted) {
+		    $up_disk_H->execute(0, $disk_id);
+		}
+		next;
+	    }
+	    if (!$mounted) {
+		$up_disk_H->execute(1, $disk_id);
+	    }
+	} else {
+	    if ($mounted) {
+		$up_disk_H->execute(0, $disk_id);
+	    }
+	    next;
+	}
+    } else {
+	next unless $mounted;
+    }
+    $disk_id{$devno} = $disk_id;
+}
+$sel_disk_H->finish;
+
+if ($update_mounts) {
+    while (my($devno, $comment) = each %mounts) {
+	next if $disk_id{$devno};
+	$ins_disk_H->execute($devno, $thishost, 1, $comment);
+	$sel_lastid_H->execute;
+	($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
+	$sel_lastid_H->finish;
+    }
+}
+
+my $start_dir = cwd();
+
+my @dirs = @ARGV;
+@dirs = '.' unless @dirs;
+foreach (@dirs) {
+    $_ = abs_path($_);
+}
+
+$| = 1;
+
+my $exit_code = 0;
+
+my $md4 = Digest::MD4->new;
+my $md5 = Digest::MD5->new;
+
+while (@dirs) {
+    my $dir = shift @dirs;
+
+    if (!chdir($dir)) {
+	warn "Unable to chdir to $dir: $!\n";
+	next;
+    }
+    if (!opendir(DP, '.')) {
+	warn "Unable to opendir $dir: $!\n";
+	next;
+    }
+
+    my $reldir = $dir;
+    $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
+    print "$reldir ... \n" if $verbosity;
+
+    my @subdirs;
+    while (defined(my $fn = readdir(DP))) {
+	next if $fn =~ /^\.\.?$/ || -l $fn;
+	if (-d _) {
+	    push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
+	    next;
+	}
+	next unless -f _;
+
+	my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
+	my $disk_id = $disk_id{$dev} or next;
+	$sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
+	my($sum4, $dbsum4, $sum5, $dbsum5);
+	my $dbsumcnt = 0;
+	while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
+	    if ($sum_type == 4) {
+		$dbsum4 = $checksum;
+		$dbsumcnt++;
+	    } elsif ($sum_type == 5) {
+		$dbsum5 = $checksum;
+		$dbsumcnt++;
+	    }
+	}
+	$sel_sum_H->finish;
+
+	next if !$check_opt && $dbsumcnt == 2;
+
+	if (!$check_opt || $dbsumcnt || $verbosity > 2) {
+	    if (!open(IN, $fn)) {
+		print STDERR "Unable to read $fn: $!\n";
+		next;
+	    }
+
+	    while (1) {
+		while (sysread(IN, $_, 64*1024)) {
+		    $md4->add($_);
+		    $md5->add($_);
+		}
+		$sum4 = $md4->digest;
+		$sum5 = $md5->digest;
+		print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
+		print " $fn" if $verbosity > 1;
+		my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
+		last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
+		$ino = $ino2;
+		$size = $size2;
+		$mtime = $mtime2;
+		$ctime = $ctime2;
+		sysseek(IN, 0, 0);
+		print " REREADING\n" if $verbosity > 1;
+	    }
+
+	    close IN;
+	} elsif ($verbosity > 1) {
+	    print "_$fn";
+	}
+
+	if ($check_opt) {
+	    my $dif;
+	    if ($dbsumcnt == 0) {
+		$dif = ' --MISSING--';
+	    } else {
+		$dif = '';
+		if (!defined $dbsum4) {
+		    $dif .= ' -NO-MD4-';
+		} elsif ($sum4 ne $dbsum4) {
+		    $dif .= ' -MD4-CHANGED-';
+		}
+		if (!defined $dbsum5) {
+		    $dif .= ' ---NO-MD5---';
+		} elsif ($sum5 ne $dbsum5) {
+		    $dif .= ' -MD5-CHANGED-';
+		}
+		if ($dif eq '') {
+		    print " ====OK====\n" if $verbosity > 1;
+		    next;
+		}
+		$dif =~ s/MD4-CHANGED MD5-//;
+	    }
+	    if ($verbosity < 2) {
+		print $verbosity ? ' ' : "$reldir/";
+		print $fn;
+	    }
+	    print $dif, "\n";
+	    $exit_code = 1;
+	} else {
+	    print "\n" if $verbosity > 1;
+	    $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
+	    $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
+	}
+    }
+
+    closedir DP;
+
+    unshift(@dirs, sort @subdirs) if $recurse_opt;
+}
+
+exit $exit_code;
+
+sub usage
+{
+    die <<EOT;
+Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
+
+Options:
+     --db=FILE     Specify the config FILE to read for the DB info.
+ -m, --mounts      Update mount info.
+ -r, --recurse     Scan files in subdirectories too.
+ -c, --check       Check if the checksums are right (doesn't update).
+ -v, --verbose     Mention what we're doing.  Repeat for more info.
+ -h, --help        Display this help message.
+EOT
+}