1 Added some DB-access routines to help rsync keep extra filesystem info
2 about the files it is dealing with. This adds both the --db=CONFIG_FILE
3 option and the "db config" daemon parameter.
5 For the moment this only adds checksum caching when the --checksum option
6 is used. Future improvements may include:
8 - Updating of MD5 checksums when transferring any file, even w/o -c.
9 We should be able to extend this to work for MD4 checksums too if we
10 make the sender force checksum_seed to 0 when using a DB and having
11 the receiving side check to see if it got a 0 checksum_seed. (We
12 probably don't want to compute 2 MD4 checksums for the case where
13 the checksum_seed is non-zero.)
15 - Caching of path info that allows for the finding of files to use for
16 moving/linking/copying/alternate-basis-use.
18 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
20 To use this patch, run these commands for a successful build:
22 patch -p1 <patches/db.diff
23 ./configure (optional if already run)
26 based-on: 3b8f8192227b14e708bf535072485e50f4362270
27 diff --git a/Makefile.in b/Makefile.in
30 @@ -36,7 +36,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
31 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
32 util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
33 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
34 - fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
35 + fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
36 OBJS3=progress.o pipe.o
37 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
38 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
39 diff --git a/checksum.c b/checksum.c
44 extern int checksum_seed;
45 extern int protocol_version;
49 a simple 32 bit checksum that can be upadted from either end
50 @@ -98,10 +99,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
54 -void file_checksum(char *fname, char *sum, OFF_T size)
55 +void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
57 struct map_struct *buf;
58 - OFF_T i, len = size;
59 + OFF_T i, len = st_p->st_size;
63 @@ -112,7 +113,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
67 - buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
68 + buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
70 if (protocol_version >= 30) {
72 @@ -146,6 +147,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
73 mdfour_result(&m, (uchar *)sum);
77 + db_set_checksum(fname, st_p, sum);
82 diff --git a/cleanup.c b/cleanup.c
90 extern int keep_partial;
91 extern int got_xfer_error;
92 extern int output_needs_newline;
93 @@ -130,6 +131,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
103 if (cleanup_child_pid != -1) {
105 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
106 diff --git a/clientserver.c b/clientserver.c
109 @@ -42,13 +42,16 @@ extern int numeric_ids;
110 extern int filesfrom_fd;
111 extern int remote_protocol;
112 extern int protocol_version;
113 +extern int always_checksum;
114 extern int io_timeout;
115 extern int no_detach;
117 extern int write_batch;
118 extern int default_af_hint;
119 extern int logfile_format_has_i;
120 extern int logfile_format_has_o_or_i;
121 extern mode_t orig_umask;
122 +extern char *db_config;
123 extern char *bind_address;
124 extern char *config_file;
125 extern char *logfile_format;
126 @@ -668,6 +671,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
130 + if (*lp_db_config(i))
131 + db_read_config(FLOG, lp_db_config(i));
134 if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
136 @@ -859,6 +865,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
138 am_server = 1; /* Don't let someone try to be tricky. */
141 + if (!always_checksum)
144 if (lp_ignore_errors(module_id))
147 diff --git a/configure.in b/configure.in
150 @@ -322,7 +322,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
151 sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
152 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
153 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
154 - popt.h popt/popt.h)
155 + popt.h popt/popt.h mysql/mysql.h sqlite3.h)
158 AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[
159 @@ -1004,6 +1004,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
163 +AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
164 +if test x$MYSQL_CONFIG = x1; then
165 + AC_MSG_CHECKING(for mysql version >= 4)
166 + mysql_version=`mysql_config --version`
167 + mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
168 + if test $mysql_major_version -lt 4; then
169 + AC_MSG_RESULT(no.. skipping MySQL)
173 + MYSQL_CFLAGS=`mysql_config --cflags`
174 + MYSQL_LIBS=`mysql_config --libs`
176 + CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
177 + LIBS="$MYSQL_LIBS $LIBS"
179 + AC_CHECK_LIB(mysqlclient, mysql_init)
183 +AC_CHECK_LIB(sqlite3, sqlite3_open)
184 +AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
187 ' checker'*|checker*)
188 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
189 diff --git a/db.c b/db.c
195 + * Routines to access extended file info via DB.
197 + * Copyright (C) 2008 Wayne Davison
199 + * This program is free software; you can redistribute it and/or modify
200 + * it under the terms of the GNU General Public License as published by
201 + * the Free Software Foundation; either version 3 of the License, or
202 + * (at your option) any later version.
204 + * This program is distributed in the hope that it will be useful,
205 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
206 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
207 + * GNU General Public License for more details.
209 + * You should have received a copy of the GNU General Public License along
210 + * with this program; if not, visit the http://fsf.org website.
217 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
219 +#include <mysql/mysql.h>
220 +#include <mysql/errmsg.h>
223 +#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
225 +#include <sqlite3.h>
226 +#ifndef HAVE_SQLITE3_OPEN_V2
227 +#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
228 + sqlite3_open(dbname, dbhptr)
230 +#ifndef HAVE_SQLITE3_PREPARE_V2
231 +#define sqlite3_prepare_v2 sqlite3_prepare
235 +extern int protocol_version;
236 +extern int checksum_len;
238 +#define DB_TYPE_NONE 0
239 +#define DB_TYPE_MYSQL 1
240 +#define DB_TYPE_SQLITE 2
242 +int use_db = DB_TYPE_NONE;
244 +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
245 +static unsigned int dbport = 0;
260 +#define MAX_PREP_CNT 3
267 + sqlite3_stmt *sqlite;
270 +} statements[MAX_PREP_CNT];
273 +static enum logcode log_code;
276 +static unsigned int bind_disk_id;
277 +static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
278 +static char bind_sum[MAX_DIGEST_LEN];
280 +static char bind_thishost[256];
281 +static int bind_thishost_len;
283 +static unsigned int prior_disk_id = 0;
284 +static unsigned long long prior_devno = 0;
286 +int db_read_config(enum logcode code, const char *config_file)
288 + char buf[2048], *cp;
294 + bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
296 + if (!(fp = fopen(config_file, "r"))) {
297 + rsyserr(log_code, errno, "unable to open %s", config_file);
300 + while (fgets(buf, sizeof buf, fp)) {
302 + if ((cp = strchr(buf, '#')) == NULL
303 + && (cp = strchr(buf, '\r')) == NULL
304 + && (cp = strchr(buf, '\n')) == NULL)
305 + cp = buf + strlen(buf);
306 + while (cp != buf && isSpace(cp-1)) cp--;
312 + if (!(cp = strchr(buf, ':')))
316 + while (isSpace(cp)) cp++;
317 + if (strcasecmp(buf, "dbhost") == 0)
318 + dbhost = strdup(cp);
319 + else if (strcasecmp(buf, "dbuser") == 0)
320 + dbuser = strdup(cp);
321 + else if (strcasecmp(buf, "dbpass") == 0)
322 + dbpass = strdup(cp);
323 + else if (strcasecmp(buf, "dbname") == 0)
324 + dbname = strdup(cp);
325 + else if (strcasecmp(buf, "dbport") == 0)
327 + else if (strcasecmp(buf, "thishost") == 0)
328 + bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
329 + else if (strcasecmp(buf, "dbtype") == 0) {
331 + if (strcasecmp(cp, "mysql") == 0) {
332 + use_db = DB_TYPE_MYSQL;
337 + if (strcasecmp(cp, "sqlite") == 0) {
338 + use_db = DB_TYPE_SQLITE;
343 + "Unsupported dbtype on line #%d in %s.\n",
344 + lineno, config_file);
345 + use_db = DB_TYPE_NONE;
349 + rprintf(log_code, "Invalid line #%d in %s\n",
350 + lineno, config_file);
351 + use_db = DB_TYPE_NONE;
357 + if (bind_thishost_len >= (int)sizeof bind_thishost)
358 + bind_thishost_len = sizeof bind_thishost - 1;
360 + if (!use_db || !dbname) {
361 + rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
362 + use_db = DB_TYPE_NONE;
366 + md_num = protocol_version >= 30 ? 5 : 4;
372 +static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
376 + int qlen, param_cnt;
377 + MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
380 + out_of_memory("prepare_mysql");
383 + qlen = vasprintf(&query, fmt, ap);
386 + out_of_memory("prepare_mysql");
388 + if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
389 + rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
394 + if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
395 + rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
396 + param_cnt, bind_cnt);
400 + mysql_stmt_bind_param(stmt, binds);
407 +static int db_connect_mysql(void)
409 + MYSQL_BIND binds[10];
411 + if (!(dbh.mysql = mysql_init(NULL)))
412 + out_of_memory("db_read_config");
414 + if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
417 + memset(binds, 0, sizeof binds);
418 + binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
419 + binds[0].buffer = &bind_devno;
420 + binds[1].buffer_type = MYSQL_TYPE_STRING;
421 + binds[1].buffer = &bind_thishost;
422 + binds[1].buffer_length = bind_thishost_len;
423 + statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
426 + " WHERE devno = ? AND host = ? AND mounted = 1");
427 + if (!statements[SEL_DEV].mysql)
430 + memset(binds, 0, sizeof binds);
431 + binds[0].buffer_type = MYSQL_TYPE_LONG;
432 + binds[0].buffer = &bind_disk_id;
433 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
434 + binds[1].buffer = &bind_ino;
435 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
436 + binds[2].buffer = &bind_size;
437 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
438 + binds[3].buffer = &bind_mtime;
439 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
440 + binds[4].buffer = &bind_ctime;
441 + statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
444 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
445 + " AND size = ? AND mtime = ? AND ctime = ?",
447 + if (!statements[SEL_SUM].mysql)
450 + memset(binds, 0, sizeof binds);
451 + binds[0].buffer_type = MYSQL_TYPE_LONG;
452 + binds[0].buffer = &bind_disk_id;
453 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
454 + binds[1].buffer = &bind_ino;
455 + binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
456 + binds[2].buffer = binds[6].buffer = &bind_size;
457 + binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
458 + binds[3].buffer = binds[7].buffer = &bind_mtime;
459 + binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
460 + binds[4].buffer = binds[8].buffer = &bind_ctime;
461 + binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
462 + binds[5].buffer = binds[9].buffer = &bind_sum;
463 + binds[5].buffer_length = binds[9].buffer_length = checksum_len;
464 + statements[REP_SUM].mysql = prepare_mysql(binds, 10,
465 + "INSERT INTO inode_map"
466 + " SET disk_id = ?, ino = ?, sum_type = %d,"
467 + " size = ?, mtime = ?, ctime = ?, checksum = ?"
468 + " ON DUPLICATE KEY"
469 + " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
471 + if (!statements[REP_SUM].mysql)
479 +static int db_connect_sqlite(void)
483 + if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
486 + sql = "SELECT disk_id"
488 + " WHERE devno = ? AND host = ? AND mounted = 1";
489 + if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
495 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
496 + " AND size = ? AND mtime = ? AND ctime = ?",
498 + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
503 + "INSERT OR REPLACE INTO inode_map"
504 + " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
505 + " VALUES(?, ?, %d, ?, ?, ?, ?)",
507 + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
515 +int db_connect(void)
519 + case DB_TYPE_MYSQL:
520 + if (db_connect_mysql())
525 + case DB_TYPE_SQLITE:
526 + if (db_connect_sqlite())
532 + rprintf(log_code, "Unable to connect to DB\n");
534 + use_db = DB_TYPE_NONE;
539 +void db_disconnect(void)
546 + for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
547 + if (statements[ndx].all) {
550 + case DB_TYPE_MYSQL:
551 + mysql_stmt_close(statements[ndx].mysql);
555 + case DB_TYPE_SQLITE:
556 + sqlite3_finalize(statements[ndx].sqlite);
560 + statements[ndx].all = NULL;
566 + case DB_TYPE_MYSQL:
567 + mysql_close(dbh.mysql);
571 + case DB_TYPE_SQLITE:
572 + sqlite3_close(dbh.sqlite);
581 +static MYSQL_STMT *exec_mysql(int ndx)
583 + MYSQL_STMT *stmt = statements[ndx].mysql;
586 + if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
588 + if (db_connect()) {
589 + stmt = statements[ndx].mysql;
590 + rc = mysql_stmt_execute(stmt);
594 + rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
603 +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
605 + unsigned long length[32];
606 + my_bool is_null[32], error[32];
611 + exit_cleanup(RERR_UNSUPPORTED);
613 + if ((stmt = exec_mysql(ndx)) == NULL)
616 + for (i = 0; i < bind_cnt; i++) {
617 + binds[i].is_null = &is_null[i];
618 + binds[i].length = &length[i];
619 + binds[i].error = &error[i];
621 + mysql_stmt_bind_result(stmt, binds);
623 + if ((rc = mysql_stmt_fetch(stmt)) != 0) {
624 + if (rc != MYSQL_NO_DATA) {
625 + rprintf(log_code, "SELECT fetch failed: %s\n",
626 + mysql_stmt_error(stmt));
628 + mysql_stmt_free_result(stmt);
632 + mysql_stmt_free_result(stmt);
634 + return is_null[0] ? 0 : 1;
638 +static void get_disk_id(unsigned long long devno)
642 + case DB_TYPE_MYSQL: {
643 + MYSQL_BIND binds[1];
645 + bind_devno = devno; /* The one variable SEL_DEV input value. */
647 + /* Bind where to put the output. */
648 + binds[0].buffer_type = MYSQL_TYPE_LONG;
649 + binds[0].buffer = &prior_disk_id;
650 + if (!fetch_mysql(binds, 1, SEL_DEV))
656 + case DB_TYPE_SQLITE: {
657 + sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
658 + sqlite3_bind_int64(stmt, 1, devno);
659 + sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
660 + if (sqlite3_step(stmt) == SQLITE_ROW)
661 + prior_disk_id = sqlite3_column_int(stmt, 0);
664 + sqlite3_reset(stmt);
670 + prior_devno = devno;
673 +int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
675 + if (prior_devno != st_p->st_dev)
676 + get_disk_id(st_p->st_dev);
677 + if (prior_disk_id == 0)
682 + case DB_TYPE_MYSQL: {
683 + MYSQL_BIND binds[1];
685 + bind_disk_id = prior_disk_id;
686 + bind_ino = st_p->st_ino;
687 + bind_size = st_p->st_size;
688 + bind_mtime = st_p->st_mtime;
689 + bind_ctime = st_p->st_ctime;
691 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
692 + binds[0].buffer = sum;
693 + binds[0].buffer_length = checksum_len;
694 + return fetch_mysql(binds, 1, SEL_SUM);
698 + case DB_TYPE_SQLITE: {
699 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
700 + sqlite3_bind_int(stmt, 1, prior_disk_id);
701 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
702 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
703 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
704 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
705 + if (sqlite3_step(stmt) == SQLITE_ROW) {
706 + int len = sqlite3_column_bytes(stmt, 0);
707 + if (len > MAX_DIGEST_LEN)
708 + len = MAX_DIGEST_LEN;
709 + memcpy(sum, sqlite3_column_blob(stmt, 0), len);
710 + sqlite3_reset(stmt);
713 + sqlite3_reset(stmt);
722 +int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
724 + if (prior_devno != st_p->st_dev)
725 + get_disk_id(st_p->st_dev);
726 + if (prior_disk_id == 0)
731 + case DB_TYPE_MYSQL: {
732 + bind_disk_id = prior_disk_id;
733 + bind_ino = st_p->st_ino;
734 + bind_size = st_p->st_size;
735 + bind_mtime = st_p->st_mtime;
736 + bind_ctime = st_p->st_ctime;
737 + memcpy(bind_sum, sum, checksum_len);
739 + return exec_mysql(REP_SUM) != NULL;
743 + case DB_TYPE_SQLITE: {
745 + sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
746 + sqlite3_bind_int(stmt, 1, prior_disk_id);
747 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
748 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
749 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
750 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
751 + sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
752 + rc = sqlite3_step(stmt);
753 + sqlite3_reset(stmt);
754 + return rc == SQLITE_DONE;
761 diff --git a/flist.c b/flist.c
764 @@ -54,6 +54,7 @@ extern int preserve_specials;
765 extern int missing_args;
769 extern int eol_nulls;
770 extern int relative_paths;
771 extern int implied_dirs;
772 @@ -1308,11 +1309,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
773 extra_len += EXTRA_LEN;
776 - if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
777 - file_checksum(thisname, tmp_sum, st.st_size);
778 - if (sender_keeps_checksum)
779 - extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
781 + if (sender_keeps_checksum && S_ISREG(st.st_mode))
782 + extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
784 #if EXTRA_ROUNDING > 0
785 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
786 @@ -1395,8 +1393,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
790 - if (sender_keeps_checksum && S_ISREG(st.st_mode))
791 - memcpy(F_SUM(file), tmp_sum, checksum_len);
792 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
793 + if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
794 + file_checksum(thisname, &st, tmp_sum);
795 + if (sender_keeps_checksum)
796 + memcpy(F_SUM(file), tmp_sum, checksum_len);
800 F_NDX(file) = stats.num_dirs;
801 @@ -2061,6 +2063,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
802 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
803 int implied_dot_dir = 0;
808 rprintf(FLOG, "building file list\n");
809 if (show_filelist_p())
810 start_filelist_progress("building file list");
811 diff --git a/generator.c b/generator.c
814 @@ -60,6 +60,7 @@ extern int human_readable;
815 extern int ignore_existing;
816 extern int ignore_non_existing;
819 extern int append_mode;
820 extern int make_backups;
821 extern int csum_length;
822 @@ -532,7 +533,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
823 of the file time to determine whether to sync */
824 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
825 char sum[MAX_DIGEST_LEN];
826 - file_checksum(fn, sum, st->st_size);
827 + if (!use_db || !db_get_checksum(fn, st, sum))
828 + file_checksum(fn, st, sum);
829 return memcmp(sum, F_SUM(file), checksum_len) == 0;
832 @@ -2074,6 +2076,9 @@ void generate_files(int f_out, const char *local_name)
836 + if (use_db && always_checksum)
839 /* Since we often fill up the outgoing socket and then just sit around
840 * waiting for the other 2 processes to do their thing, we don't want
841 * to exit on a timeout. If the data stops flowing, the receiver will
842 diff --git a/loadparm.c b/loadparm.c
845 @@ -108,6 +108,7 @@ typedef struct {
853 @@ -182,6 +183,7 @@ static const all_vars Defaults = {
854 /* auth_users; */ NULL,
857 + /* db_config; */ NULL,
858 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
860 /* exclude_from; */ NULL,
861 @@ -317,6 +319,7 @@ static struct parm_struct parm_table[] =
862 {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0},
863 {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0},
864 {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0},
865 + {"db config", P_STRING, P_LOCAL, &Vars.l.db_config, NULL,0},
866 {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0},
867 {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0},
868 {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0},
869 @@ -447,6 +450,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
870 FN_LOCAL_STRING(lp_auth_users, auth_users)
871 FN_LOCAL_STRING(lp_charset, charset)
872 FN_LOCAL_STRING(lp_comment, comment)
873 +FN_LOCAL_STRING(lp_db_config, db_config)
874 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
875 FN_LOCAL_STRING(lp_exclude, exclude)
876 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
877 diff --git a/main.c b/main.c
880 @@ -50,6 +50,7 @@ extern int copy_unsafe_links;
881 extern int keep_dirlinks;
882 extern int preserve_hard_links;
883 extern int protocol_version;
884 +extern int always_checksum;
885 extern int file_total;
887 extern int xfer_dirs;
888 @@ -79,6 +80,7 @@ extern char *filesfrom_host;
889 extern char *partial_dir;
890 extern char *dest_option;
891 extern char *rsync_path;
892 +extern char *db_config;
893 extern char *shell_cmd;
894 extern char *batch_name;
895 extern char *password_file;
896 @@ -1584,6 +1586,9 @@ int main(int argc,char *argv[])
897 exit_cleanup(RERR_SYNTAX);
900 + if (db_config && always_checksum)
901 + db_read_config(FERROR, db_config);
904 set_nonblocking(STDIN_FILENO);
905 set_nonblocking(STDOUT_FILENO);
906 diff --git a/options.c b/options.c
909 @@ -92,6 +92,7 @@ int use_qsort = 0;
910 char *files_from = NULL;
911 int filesfrom_fd = -1;
912 char *filesfrom_host = NULL;
913 +char *db_config = NULL;
915 int protect_args = 0;
916 int human_readable = 1;
917 @@ -567,6 +568,7 @@ static void print_rsync_version(enum logcode f)
918 char const *links = "no ";
919 char const *iconv = "no ";
920 char const *ipv6 = "no ";
921 + char const *db = "no ";
922 STRUCT_STAT *dumstat;
924 #if SUBPROTOCOL_VERSION != 0
925 @@ -600,6 +602,11 @@ static void print_rsync_version(enum logcode f)
926 #ifdef CAN_SET_SYMLINK_TIMES
929 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
931 +#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
935 rprintf(f, "%s version %s protocol version %d%s\n",
936 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
937 @@ -613,8 +620,8 @@ static void print_rsync_version(enum logcode f)
938 (int)(sizeof (int64) * 8));
939 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
940 got_socketpair, hardlinks, links, ipv6, have_inplace);
941 - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
942 - have_inplace, acls, xattrs, iconv, symtimes);
943 + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n",
944 + have_inplace, acls, xattrs, iconv, symtimes, db);
946 #ifdef MAINTAINER_MODE
947 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
948 @@ -662,6 +669,7 @@ void usage(enum logcode F)
949 rprintf(F," -q, --quiet suppress non-error messages\n");
950 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
951 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
952 + rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
953 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
954 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
955 rprintf(F," -r, --recursive recurse into directories\n");
956 @@ -934,6 +942,7 @@ static struct poptOption long_options[] = {
957 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
958 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
959 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
960 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
961 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
962 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
963 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
964 diff --git a/pipe.c b/pipe.c
967 @@ -27,6 +27,9 @@ extern int am_server;
968 extern int blocking_io;
969 extern int filesfrom_fd;
970 extern int munge_symlinks;
971 +extern int always_checksum;
973 +extern char *db_config;
974 extern mode_t orig_umask;
975 extern char *logfile_name;
976 extern int remote_option_cnt;
977 @@ -143,6 +146,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
984 if (remote_option_cnt) {
985 int rc = remote_option_cnt + 1;
986 const char **rv = remote_options;
987 @@ -150,6 +156,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
989 exit_cleanup(RERR_SYNTAX);
991 + if (db_config && always_checksum)
992 + db_read_config(FERROR, db_config);
995 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
996 diff --git a/rsync.yo b/rsync.yo
999 @@ -323,6 +323,7 @@ to the detailed description below for a complete description. verb(
1000 -q, --quiet suppress non-error messages
1001 --no-motd suppress daemon-mode MOTD (see caveat)
1002 -c, --checksum skip based on checksum, not mod-time & size
1003 + --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
1004 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
1005 --no-OPTION turn off an implied OPTION (e.g. --no-D)
1006 -r, --recursive recurse into directories
1007 @@ -587,6 +588,47 @@ option's before-the-transfer "Does this file need to be updated?" check.
1008 For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
1009 MD5. For older protocols, the checksum used is MD4.
1011 +dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
1012 +that holds connection details for a database of checksum information.
1013 +When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
1014 +use cached checksum information from the DB, and will update it if it is
1017 +The currently supported DB choices are MySQL and SQLite. For example, a
1018 +MySQL configuration might look like this:
1020 +verb( dbtype: mysql
1026 + thishost: hostname )
1028 +And a SQLite configuration might look like this:
1030 +verb( dbtype: SQLite
1031 + dbname: /var/cache/rsync/sum.db )
1033 +This option only affects one side of a transfer. See the
1034 +bf(--remote-option) option for a way to specify the option for both
1035 +sides of the transfer (with each side reading the config file from
1036 +their local filesystem). For example:
1038 +verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
1040 +See the perl script "rsyncdb" in the support directory of the source code
1041 +(which may also be installed in /usr/bin) for a way to create the tables,
1042 +populate the mounted-disk information, check files against their checksums,
1043 +and update both the MD4 and MD5 checksums for files at the same time (since
1044 +an rsync copy will only update one or the other).
1046 +You can use a single MySQL DB for all your hosts if you give each one
1047 +their own "thishost" name and setup their device-mapping data. Or feel
1048 +free to use separate databases, separate servers, etc. See the rsync
1049 +daemon's "db config" parameter for how to configure a daemon to use a DB
1050 +(since a client cannot control this parameter on a daemon).
1052 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
1053 way of saying you want recursion and want to preserve almost
1054 everything (with -H being a notable omission).
1055 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
1056 --- a/rsyncd.conf.yo
1057 +++ b/rsyncd.conf.yo
1058 @@ -301,6 +301,18 @@ is daemon. This setting has no effect if the "log file" setting is a
1059 non-empty string (either set in the per-modules settings, or inherited
1060 from the global settings).
1062 +dit(bf(db config)) This parameter specifies a config file to read that
1063 +holds connection details for a database of checksum information.
1065 +The config file will be read-in prior to any chroot restrictions, but
1066 +the connection occurs from inside the chroot. This means that you
1067 +should use a socket connection (e.g. 127.0.0.1 rather than localhost)
1068 +for a MySQL config from inside a chroot. For SQLite, the DB file must
1069 +be placed inside the chroot (though it can be placed outside the
1070 +transfer dir if you configured an inside-chroot path).
1072 +See the bf(--db=CONFIG_FILE) option for full details.
1074 dit(bf(max verbosity)) This parameter allows you to control
1075 the maximum amount of verbose information that you'll allow the daemon to
1076 generate (since the information goes into the log file). The default is 1,
1077 diff --git a/support/rsyncdb b/support/rsyncdb
1078 new file mode 100755
1080 +++ b/support/rsyncdb
1087 +use Cwd qw(abs_path cwd);
1091 +my $MOUNT_FILE = '/etc/mtab';
1093 +&Getopt::Long::Configure('bundling');
1094 +&usage if !&GetOptions(
1095 + 'db=s' => \( my $db_config ),
1096 + 'init' => \( my $init_db ),
1097 + 'mounts|m' => \( my $update_mounts ),
1098 + 'recurse|r' => \( my $recurse_opt ),
1099 + 'check|c' => \( my $check_opt ),
1100 + 'verbose|v+' => \( my $verbosity = 0 ),
1101 + 'help|h' => \( my $help_opt ),
1103 +&usage if $help_opt || !defined $db_config;
1106 +open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
1110 + my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
1111 + $config{$key} = $val;
1115 +die "You must define at least dbtype and dbname in $db_config\n"
1116 + unless defined $config{'dbtype'} && defined $config{'dbname'};
1118 +my $sqlite = $config{'dbtype'} =~ /^sqlite$/i;
1120 +my $thishost = $config{'thishost'} || 'localhost';
1122 +my $connect = 'DBI:' . $config{'dbtype'} . ':';
1123 +$connect .= 'dbname=' . $config{'dbname'} if $sqlite;
1124 +$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db;
1125 +$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
1126 +$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
1128 +my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
1129 + or die "DB connection failed\n";
1132 + $dbh->disconnect if defined $dbh;
1136 + my $unsigned = $sqlite ? '' : 'unsigned';
1137 + my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT';
1138 + my $dbname = $config{'dbname'};
1141 + $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`");
1142 + $dbh->do("USE `$dbname`");
1145 + print "Dropping old tables (if they exist) ...\n" if $verbosity;
1146 + $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr;
1147 + $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr;
1149 + print "Creating empty tables ...\n" if $verbosity;
1151 + CREATE TABLE disk (
1152 + disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment,
1153 + devno bigint $unsigned NOT NULL,
1154 + host varchar(256) NOT NULL default 'localhost',
1155 + mounted tinyint NOT NULL default '1',
1156 + comment varchar(256) default NULL
1157 + )") or die $dbh->errstr;
1160 + CREATE TABLE inode_map (
1161 + disk_id integer $unsigned NOT NULL,
1162 + ino bigint $unsigned NOT NULL,
1163 + size bigint $unsigned NOT NULL,
1164 + mtime bigint NOT NULL,
1165 + ctime bigint NOT NULL,
1166 + sum_type tinyint NOT NULL default '0',
1167 + checksum binary(16) NOT NULL,
1168 + PRIMARY KEY (disk_id,ino,sum_type)
1169 + )") or die $dbh->errstr;
1171 + exit unless $update_mounts;
1174 +my $sel_disk_H = $dbh->prepare("
1175 + SELECT disk_id, devno, mounted, comment
1178 + ") or die $dbh->errstr;
1180 +my $ins_disk_H = $dbh->prepare("
1182 + (devno, host, mounted, comment)
1183 + VALUES(?, ?, ?, ?)
1184 + ") or die $dbh->errstr;
1186 +my $up_disk_H = $dbh->prepare("
1190 + ") or die $dbh->errstr;
1192 +my $row_id = $sqlite ? 'ROWID' : 'ID';
1193 +my $sel_lastid_H = $dbh->prepare("
1194 + SELECT LAST_INSERT_$row_id()
1195 + ") or die $dbh->errstr;
1197 +my $sel_sum_H = $dbh->prepare("
1198 + SELECT sum_type, checksum
1200 + WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
1201 + ") or die $dbh->errstr;
1203 +my $rep_sum_H = $dbh->prepare("
1204 + REPLACE INTO inode_map
1205 + (disk_id, ino, size, mtime, ctime, sum_type, checksum)
1206 + VALUES(?, ?, ?, ?, ?, ?, ?)
1207 + ") or die $dbh->errstr;
1210 +if ($update_mounts) {
1211 + open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
1213 + my($devname, $mnt) = (split)[0,1];
1214 + next unless $devname =~ m#^/dev#;
1215 + my($devno) = (stat($mnt))[0];
1216 + if (!defined $devno) {
1217 + warn "Unable to stat $mnt: $!\n";
1220 + $mounts{$devno} = "$devname on $mnt";
1226 +$sel_disk_H->execute($thishost);
1227 +while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
1228 + if ($update_mounts) {
1229 + if (defined $mounts{$devno}) {
1230 + if ($comment ne $mounts{$devno}) {
1232 + print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
1233 + $up_disk_H->execute(0, $disk_id);
1238 + print "Mounting $comment ($thishost:$devno)\n" if $verbosity;
1239 + $up_disk_H->execute(1, $disk_id);
1243 + print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
1244 + $up_disk_H->execute(0, $disk_id);
1249 + next unless $mounted;
1251 + $disk_id{$devno} = $disk_id;
1253 +$sel_disk_H->finish;
1255 +if ($update_mounts) {
1256 + while (my($devno, $comment) = each %mounts) {
1257 + next if $disk_id{$devno};
1258 + print "Adding $comment ($thishost:$devno)\n" if $verbosity;
1259 + $ins_disk_H->execute($devno, $thishost, 1, $comment);
1260 + $sel_lastid_H->execute;
1261 + ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
1262 + $sel_lastid_H->finish;
1267 +my $start_dir = cwd();
1270 +@dirs = '.' unless @dirs;
1272 + $_ = abs_path($_);
1279 +my $md4 = Digest::MD4->new;
1280 +my $md5 = Digest::MD5->new;
1283 + my $dir = shift @dirs;
1285 + if (!chdir($dir)) {
1286 + warn "Unable to chdir to $dir: $!\n";
1289 + if (!opendir(DP, '.')) {
1290 + warn "Unable to opendir $dir: $!\n";
1294 + my $reldir = $dir;
1295 + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
1296 + print "$reldir ... \n" if $verbosity;
1299 + while (defined(my $fn = readdir(DP))) {
1300 + next if $fn =~ /^\.\.?$/ || -l $fn;
1302 + push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
1307 + my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
1308 + my $disk_id = $disk_id{$dev} or next;
1309 + $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
1310 + my($sum4, $dbsum4, $sum5, $dbsum5);
1312 + while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
1313 + if ($sum_type == 4) {
1314 + $dbsum4 = $checksum;
1316 + } elsif ($sum_type == 5) {
1317 + $dbsum5 = $checksum;
1321 + $sel_sum_H->finish;
1323 + next if !$check_opt && $dbsumcnt == 2;
1325 + if (!$check_opt || $dbsumcnt || $verbosity > 2) {
1326 + if (!open(IN, $fn)) {
1327 + print STDERR "Unable to read $fn: $!\n";
1332 + while (sysread(IN, $_, 64*1024)) {
1336 + $sum4 = $md4->digest;
1337 + $sum5 = $md5->digest;
1338 + print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
1339 + print " $fn" if $verbosity > 1;
1340 + my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
1341 + last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
1346 + sysseek(IN, 0, 0);
1347 + print " REREADING\n" if $verbosity > 1;
1351 + } elsif ($verbosity > 1) {
1357 + if ($dbsumcnt == 0) {
1358 + $dif = ' --MISSING--';
1361 + if (!defined $dbsum4) {
1362 + $dif .= ' -NO-MD4-';
1363 + } elsif ($sum4 ne $dbsum4) {
1364 + $dif .= ' -MD4-CHANGED-';
1366 + if (!defined $dbsum5) {
1367 + $dif .= ' ---NO-MD5---';
1368 + } elsif ($sum5 ne $dbsum5) {
1369 + $dif .= ' -MD5-CHANGED-';
1372 + print " ====OK====\n" if $verbosity > 1;
1375 + $dif =~ s/MD4-CHANGED MD5-//;
1377 + if ($verbosity < 2) {
1378 + print $verbosity ? ' ' : "$reldir/";
1384 + print "\n" if $verbosity > 1;
1385 + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
1386 + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
1392 + unshift(@dirs, sort @subdirs) if $recurse_opt;
1400 +Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
1403 + --db=FILE Specify the config FILE to read for the DB info.
1404 + --init Create (recreate) needed tables (making them empty).
1405 + No DIR scanning, but can be combined with --mounts.
1406 + -m, --mounts Update mount info. Does no DIR scanning.
1407 + -r, --recurse Scan files in subdirectories too.
1408 + -c, --check Check if the checksums are right (doesn't update).
1409 + -v, --verbose Mention what we're doing. Repeat for more info.
1410 + -h, --help Display this help message.