1 Added some DB-access routines to help rsync keep extra filesystem info
2 about the files it is dealing with. This adds both the --db=CONFIG_FILE
3 option and the "db config" daemon parameter.
5 For the moment this only adds checksum caching when the --checksum option
6 is used. Future improvments may include:
8 - Updating of MD5 checksums when transferring any file, even w/o -c.
10 - Caching of path info that allows for the finding of files to use for
11 moving/linking/copying/alternate-basis-use.
13 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
15 To use this patch, run these commands for a successful build:
17 patch -p1 <patches/remote-option.diff
18 patch -p1 <patches/db.diff
19 ./configure (optional if already run)
22 diff --git a/Makefile.in b/Makefile.in
25 @@ -35,7 +35,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
26 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
27 util.o main.o checksum.o match.o syscall.o log.o backup.o
28 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
29 - fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
30 + fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
31 OBJS3=progress.o pipe.o
32 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
33 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
34 diff --git a/checksum.c b/checksum.c
39 extern int checksum_seed;
40 extern int protocol_version;
43 int csum_length = SHORT_SUM_LENGTH; /* initial value */
45 @@ -100,10 +101,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
49 -void file_checksum(char *fname, char *sum, OFF_T size)
50 +void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
52 struct map_struct *buf;
53 - OFF_T i, len = size;
54 + OFF_T i, len = st_p->st_size;
58 @@ -114,7 +115,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
62 - buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
63 + buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
65 if (protocol_version >= 30) {
67 @@ -148,6 +149,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
68 mdfour_result(&m, (uchar *)sum);
72 + db_set_checksum(fname, st_p, sum);
77 diff --git a/cleanup.c b/cleanup.c
80 @@ -27,6 +27,7 @@ extern int am_daemon;
82 extern int keep_partial;
83 extern int got_xfer_error;
85 extern char *partial_dir;
86 extern char *logfile_name;
88 @@ -124,6 +125,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
98 if (cleanup_child_pid != -1) {
100 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
101 diff --git a/clientserver.c b/clientserver.c
104 @@ -42,6 +42,7 @@ extern int numeric_ids;
105 extern int filesfrom_fd;
106 extern int remote_protocol;
107 extern int protocol_version;
108 +extern int always_checksum;
109 extern int io_timeout;
110 extern int no_detach;
111 extern int write_batch;
112 @@ -49,6 +50,7 @@ extern int default_af_hint;
113 extern int logfile_format_has_i;
114 extern int logfile_format_has_o_or_i;
115 extern mode_t orig_umask;
116 +extern char *db_config;
117 extern char *bind_address;
118 extern char *sockopts;
119 extern char *config_file;
120 @@ -782,6 +784,12 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
121 } else if (am_root < 0) /* Treat --fake-super from client as --super. */
124 + db_config = lp_db_config(i);
125 + if (!*db_config || (!always_checksum && protocol_version < 30))
128 + db_read_config(FLOG, db_config);
130 if (filesfrom_fd == 0)
133 diff --git a/configure.in b/configure.in
136 @@ -969,6 +969,8 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
140 +LIBS="$LIBS -lmysqlclient -lsqlite3"
143 ' checker'*|checker*)
144 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
145 diff --git a/db.c b/db.c
151 + * Routines to access extended file info via DB.
153 + * Copyright (C) 2008 Wayne Davison
155 + * This program is free software; you can redistribute it and/or modify
156 + * it under the terms of the GNU General Public License as published by
157 + * the Free Software Foundation; either version 3 of the License, or
158 + * (at your option) any later version.
160 + * This program is distributed in the hope that it will be useful,
161 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
162 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
163 + * GNU General Public License for more details.
165 + * You should have received a copy of the GNU General Public License along
166 + * with this program; if not, visit the http://fsf.org website.
176 +#include <mysql/mysql.h>
177 +#include <mysql/errmsg.h>
180 +#include <sqlite3.h>
183 +extern int protocol_version;
184 +extern int checksum_len;
186 +#define DB_TYPE_NONE 0
187 +#define DB_TYPE_MYSQL 1
188 +#define DB_TYPE_SQLITE 2
190 +int use_db = DB_TYPE_NONE;
192 +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
193 +static unsigned int dbport = 0;
208 +#define MAX_PREP_CNT 3
215 + sqlite3_stmt *sqlite;
218 +} statements[MAX_PREP_CNT];
221 +static enum logcode log_code;
223 +static unsigned int bind_disk_id;
224 +static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
225 +static char bind_thishost[256], bind_sum[MAX_DIGEST_LEN];
226 +static int bind_thishost_len;
228 +static unsigned int prior_disk_id = 0;
229 +static unsigned long long prior_devno = 0;
231 +int db_read_config(enum logcode code, const char *config_file)
233 + char buf[2048], *cp;
239 + bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
241 + if (!(fp = fopen(config_file, "r"))) {
242 + rsyserr(log_code, errno, "unable to open %s", config_file);
245 + while (fgets(buf, sizeof buf, fp)) {
247 + if ((cp = strchr(buf, '#')) == NULL
248 + && (cp = strchr(buf, '\r')) == NULL
249 + && (cp = strchr(buf, '\n')) == NULL)
250 + cp = buf + strlen(buf);
251 + while (cp != buf && isSpace(cp-1)) cp--;
257 + if (!(cp = strchr(buf, ':')))
261 + while (isSpace(cp)) cp++;
262 + if (strcasecmp(buf, "dbhost") == 0)
263 + dbhost = strdup(cp);
264 + else if (strcasecmp(buf, "dbuser") == 0)
265 + dbuser = strdup(cp);
266 + else if (strcasecmp(buf, "dbpass") == 0)
267 + dbpass = strdup(cp);
268 + else if (strcasecmp(buf, "dbname") == 0)
269 + dbname = strdup(cp);
270 + else if (strcasecmp(buf, "dbport") == 0)
272 + else if (strcasecmp(buf, "thishost") == 0)
273 + bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
274 + else if (strcasecmp(buf, "dbtype") == 0) {
276 + if (strcasecmp(cp, "mysql") == 0) {
277 + use_db = DB_TYPE_MYSQL;
282 + if (strcasecmp(cp, "sqlite") == 0) {
283 + use_db = DB_TYPE_SQLITE;
288 + "Unsupported dbtype on line #%d in %s.\n",
289 + lineno, config_file);
290 + use_db = DB_TYPE_NONE;
294 + rprintf(log_code, "Invalid line #%d in %s\n",
295 + lineno, config_file);
296 + use_db = DB_TYPE_NONE;
302 + if (bind_thishost_len >= (int)sizeof bind_thishost)
303 + bind_thishost_len = sizeof bind_thishost - 1;
305 + if (!use_db || !dbname) {
306 + rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
307 + use_db = DB_TYPE_NONE;
311 + md_num = protocol_version >= 30 ? 5 : 4;
317 +static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
321 + int qlen, param_cnt;
322 + MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
325 + out_of_memory("prepare_mysql");
328 + qlen = vasprintf(&query, fmt, ap);
331 + out_of_memory("prepare_mysql");
333 + if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
334 + rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
339 + if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
340 + rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
341 + param_cnt, bind_cnt);
345 + mysql_stmt_bind_param(stmt, binds);
352 +static int db_connect_mysql(void)
354 + MYSQL_BIND binds[10];
356 + if (!(dbh.mysql = mysql_init(NULL)))
357 + out_of_memory("db_read_config");
359 + if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
362 + memset(binds, 0, sizeof binds);
363 + binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
364 + binds[0].buffer = &bind_devno;
365 + binds[1].buffer_type = MYSQL_TYPE_STRING;
366 + binds[1].buffer = &bind_thishost;
367 + binds[1].buffer_length = bind_thishost_len;
368 + statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
371 + " WHERE devno = ? AND host = ? AND mounted = 1");
372 + if (!statements[SEL_DEV].mysql)
375 + memset(binds, 0, sizeof binds);
376 + binds[0].buffer_type = MYSQL_TYPE_LONG;
377 + binds[0].buffer = &bind_disk_id;
378 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
379 + binds[1].buffer = &bind_ino;
380 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
381 + binds[2].buffer = &bind_size;
382 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
383 + binds[3].buffer = &bind_mtime;
384 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
385 + binds[4].buffer = &bind_ctime;
386 + statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
389 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
390 + " AND size = ? AND mtime = ? AND ctime = ?",
392 + if (!statements[SEL_SUM].mysql)
395 + memset(binds, 0, sizeof binds);
396 + binds[0].buffer_type = MYSQL_TYPE_LONG;
397 + binds[0].buffer = &bind_disk_id;
398 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
399 + binds[1].buffer = &bind_ino;
400 + binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
401 + binds[2].buffer = binds[6].buffer = &bind_size;
402 + binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
403 + binds[3].buffer = binds[7].buffer = &bind_mtime;
404 + binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
405 + binds[4].buffer = binds[8].buffer = &bind_ctime;
406 + binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
407 + binds[5].buffer = binds[9].buffer = &bind_sum;
408 + binds[5].buffer_length = binds[9].buffer_length = checksum_len;
409 + statements[REP_SUM].mysql = prepare_mysql(binds, 10,
410 + "INSERT INTO inode_map"
411 + " SET disk_id = ?, ino = ?, sum_type = %d,"
412 + " size = ?, mtime = ?, ctime = ?, checksum = ?"
413 + " ON DUPLICATE KEY"
414 + " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
416 + if (!statements[REP_SUM].mysql)
424 +static int db_connect_sqlite(void)
429 + if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
432 + if (sqlite3_open(dbname, &dbh.sqlite) != 0)
436 + sql = "SELECT disk_id"
438 + " WHERE devno = ? AND host = ? AND mounted = 1";
439 + if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
445 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
446 + " AND size = ? AND mtime = ? AND ctime = ?",
448 + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
453 + "INSERT OR REPLACE INTO inode_map"
454 + " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
455 + " VALUES(?, ?, %d, ?, ?, ?, ?)",
457 + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
465 +int db_connect(void)
469 + case DB_TYPE_MYSQL:
470 + if (db_connect_mysql())
475 + case DB_TYPE_SQLITE:
476 + if (db_connect_sqlite())
482 + rprintf(log_code, "Unable to connect to DB\n");
484 + use_db = DB_TYPE_NONE;
489 +void db_disconnect(void)
496 + for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
497 + if (statements[ndx].all) {
500 + case DB_TYPE_MYSQL:
501 + mysql_stmt_close(statements[ndx].mysql);
505 + case DB_TYPE_SQLITE:
506 + sqlite3_finalize(statements[ndx].sqlite);
510 + statements[ndx].all = NULL;
516 + case DB_TYPE_MYSQL:
517 + mysql_close(dbh.mysql);
521 + case DB_TYPE_SQLITE:
522 + sqlite3_close(dbh.sqlite);
530 +static MYSQL_STMT *exec_mysql(int ndx)
532 + MYSQL_STMT *stmt = statements[ndx].mysql;
535 + if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
537 + if (db_connect()) {
538 + stmt = statements[ndx].mysql;
539 + rc = mysql_stmt_execute(stmt);
543 + rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
550 +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
552 + unsigned long length[32];
553 + my_bool is_null[32], error[32];
558 + exit_cleanup(RERR_UNSUPPORTED);
560 + if ((stmt = exec_mysql(ndx)) == NULL)
563 + for (i = 0; i < bind_cnt; i++) {
564 + binds[i].is_null = &is_null[i];
565 + binds[i].length = &length[i];
566 + binds[i].error = &error[i];
568 + mysql_stmt_bind_result(stmt, binds);
570 + if ((rc = mysql_stmt_fetch(stmt)) != 0) {
571 + if (rc != MYSQL_NO_DATA) {
572 + rprintf(log_code, "SELECT fetch failed: %s\n",
573 + mysql_stmt_error(stmt));
575 + mysql_stmt_free_result(stmt);
579 + mysql_stmt_free_result(stmt);
581 + return is_null[0] ? 0 : 1;
584 +static void get_disk_id(unsigned long long devno)
588 + case DB_TYPE_MYSQL: {
589 + MYSQL_BIND binds[1];
591 + bind_devno = devno; /* The one variable SEL_DEV input value. */
593 + /* Bind where to put the output. */
594 + binds[0].buffer_type = MYSQL_TYPE_LONG;
595 + binds[0].buffer = &prior_disk_id;
596 + if (!fetch_mysql(binds, 1, SEL_DEV))
602 + case DB_TYPE_SQLITE: {
603 + sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
604 + sqlite3_bind_int64(stmt, 1, devno);
605 + sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
606 + if (sqlite3_step(stmt) == SQLITE_ROW)
607 + prior_disk_id = sqlite3_column_int(stmt, 0);
610 + sqlite3_reset(stmt);
616 + prior_devno = devno;
619 +int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
621 + if (prior_devno != st_p->st_dev)
622 + get_disk_id(st_p->st_dev);
623 + if (prior_disk_id == 0)
628 + case DB_TYPE_MYSQL: {
629 + MYSQL_BIND binds[1];
631 + bind_disk_id = prior_disk_id;
632 + bind_ino = st_p->st_ino;
633 + bind_size = st_p->st_size;
634 + bind_mtime = st_p->st_mtime;
635 + bind_ctime = st_p->st_ctime;
637 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
638 + binds[0].buffer = sum;
639 + binds[0].buffer_length = checksum_len;
640 + return fetch_mysql(binds, 1, SEL_SUM);
644 + case DB_TYPE_SQLITE: {
645 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
646 + sqlite3_bind_int(stmt, 1, prior_disk_id);
647 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
648 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
649 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
650 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
651 + if (sqlite3_step(stmt) == SQLITE_ROW) {
652 + int len = sqlite3_column_bytes(stmt, 0);
653 + if (len > MAX_DIGEST_LEN)
654 + len = MAX_DIGEST_LEN;
655 + memcpy(sum, sqlite3_column_blob(stmt, 0), len);
656 + sqlite3_reset(stmt);
659 + sqlite3_reset(stmt);
668 +int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
670 + if (prior_devno != st_p->st_dev)
671 + get_disk_id(st_p->st_dev);
672 + if (prior_disk_id == 0)
677 + case DB_TYPE_MYSQL: {
678 + bind_disk_id = prior_disk_id;
679 + bind_ino = st_p->st_ino;
680 + bind_size = st_p->st_size;
681 + bind_mtime = st_p->st_mtime;
682 + bind_ctime = st_p->st_ctime;
683 + memcpy(bind_sum, sum, checksum_len);
685 + return exec_mysql(REP_SUM) != NULL;
689 + case DB_TYPE_SQLITE: {
691 + sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
692 + sqlite3_bind_int(stmt, 1, prior_disk_id);
693 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
694 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
695 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
696 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
697 + sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
698 + rc = sqlite3_step(stmt);
699 + sqlite3_reset(stmt);
700 + return rc == SQLITE_DONE;
707 diff --git a/flist.c b/flist.c
710 @@ -54,6 +54,7 @@ extern int preserve_devices;
711 extern int preserve_specials;
715 extern int eol_nulls;
716 extern int relative_paths;
717 extern int implied_dirs;
718 @@ -1235,14 +1236,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
719 memcpy(bp + basename_len, linkname, linkname_len);
722 - if (always_checksum && am_sender && S_ISREG(st.st_mode))
723 - file_checksum(thisname, tmp_sum, st.st_size);
726 F_PATHNAME(file) = pathname;
728 F_DEPTH(file) = extra_len / EXTRA_LEN;
730 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
731 + if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
732 + file_checksum(thisname, &st, tmp_sum);
735 /* This code is only used by the receiver when it is building
736 * a list of files for a delete pass. */
737 if (keep_dirlinks && linkname_len && flist) {
738 @@ -1858,6 +1861,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
739 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
740 int implied_dot_dir = 0;
745 rprintf(FLOG, "building file list\n");
746 if (show_filelist_p())
747 start_filelist_progress("building file list");
748 diff --git a/generator.c b/generator.c
751 @@ -58,6 +58,7 @@ extern int update_only;
752 extern int ignore_existing;
753 extern int ignore_non_existing;
756 extern int append_mode;
757 extern int make_backups;
758 extern int csum_length;
759 @@ -718,7 +719,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
760 of the file time to determine whether to sync */
761 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
762 char sum[MAX_DIGEST_LEN];
763 - file_checksum(fn, sum, st->st_size);
764 + if (!use_db || !db_get_checksum(fn, st, sum))
765 + file_checksum(fn, st, sum);
766 return memcmp(sum, F_SUM(file), checksum_len) == 0;
769 @@ -2161,6 +2163,9 @@ void generate_files(int f_out, const char *local_name)
773 + if (use_db && always_checksum)
776 /* Since we often fill up the outgoing socket and then just sit around
777 * waiting for the other 2 processes to do their thing, we don't want
778 * to exit on a timeout. If the data stops flowing, the receiver will
779 diff --git a/loadparm.c b/loadparm.c
782 @@ -126,6 +126,7 @@ typedef struct
790 @@ -177,6 +178,7 @@ static service sDefault =
791 /* auth_users; */ NULL,
794 + /* db_config; */ NULL,
795 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
797 /* exclude_from; */ NULL,
798 @@ -307,6 +309,7 @@ static struct parm_struct parm_table[] =
799 {"auth users", P_STRING, P_LOCAL, &sDefault.auth_users, NULL,0},
800 {"charset", P_STRING, P_LOCAL, &sDefault.charset, NULL,0},
801 {"comment", P_STRING, P_LOCAL, &sDefault.comment, NULL,0},
802 + {"db config", P_STRING, P_LOCAL, &sDefault.db_config, NULL,0},
803 {"dont compress", P_STRING, P_LOCAL, &sDefault.dont_compress, NULL,0},
804 {"exclude from", P_STRING, P_LOCAL, &sDefault.exclude_from, NULL,0},
805 {"exclude", P_STRING, P_LOCAL, &sDefault.exclude, NULL,0},
806 @@ -400,6 +403,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Globals.rsync_port)
807 FN_LOCAL_STRING(lp_auth_users, auth_users)
808 FN_LOCAL_STRING(lp_charset, charset)
809 FN_LOCAL_STRING(lp_comment, comment)
810 +FN_LOCAL_STRING(lp_db_config, db_config)
811 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
812 FN_LOCAL_STRING(lp_exclude, exclude)
813 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
814 diff --git a/main.c b/main.c
817 @@ -49,6 +49,7 @@ extern int copy_unsafe_links;
818 extern int keep_dirlinks;
819 extern int preserve_hard_links;
820 extern int protocol_version;
821 +extern int always_checksum;
822 extern int file_total;
824 extern int xfer_dirs;
825 @@ -73,6 +74,7 @@ extern char *partial_dir;
826 extern char *dest_option;
827 extern char *basis_dir[];
828 extern char *rsync_path;
829 +extern char *db_config;
830 extern char *shell_cmd;
831 extern char *batch_name;
832 extern char *password_file;
833 @@ -1482,6 +1484,9 @@ int main(int argc,char *argv[])
834 exit_cleanup(RERR_SYNTAX);
837 + if (db_config && (always_checksum || protocol_version >= 30))
838 + db_read_config(FERROR, db_config);
841 set_nonblocking(STDIN_FILENO);
842 set_nonblocking(STDOUT_FILENO);
843 diff --git a/options.c b/options.c
846 @@ -92,6 +92,7 @@ int use_qsort = 0;
847 char *files_from = NULL;
848 int filesfrom_fd = -1;
849 char *filesfrom_host = NULL;
850 +char *db_config = NULL;
852 int protect_args = 0;
853 int human_readable = 0;
854 @@ -321,6 +322,7 @@ void usage(enum logcode F)
855 rprintf(F," -q, --quiet suppress non-error messages\n");
856 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
857 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
858 + rprintf(F," --db=CONFIG_FILE specify a config file for FS DB\n");
859 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
860 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
861 rprintf(F," -r, --recursive recurse into directories\n");
862 @@ -579,6 +581,7 @@ static struct poptOption long_options[] = {
863 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
864 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
865 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
866 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
867 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
868 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
869 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
870 diff --git a/pipe.c b/pipe.c
873 @@ -26,6 +26,10 @@ extern int am_sender;
874 extern int am_server;
875 extern int blocking_io;
876 extern int filesfrom_fd;
877 +extern int always_checksum;
878 +extern int protocol_version;
880 +extern char *db_config;
881 extern mode_t orig_umask;
882 extern char *logfile_name;
883 extern int remote_option_cnt;
884 @@ -141,6 +145,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
891 if (remote_option_cnt) {
892 int rc = remote_option_cnt + 1;
893 const char **rv = remote_options;
894 @@ -148,6 +155,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
896 exit_cleanup(RERR_SYNTAX);
898 + if (db_config && (always_checksum || protocol_version >= 30))
899 + db_read_config(FERROR, db_config);
902 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
903 diff --git a/receiver.c b/receiver.c
906 @@ -43,11 +43,13 @@ extern int basis_dir_cnt;
907 extern int make_backups;
908 extern int cleanup_got_literal;
909 extern int remove_source_files;
910 +extern int always_checksum;
911 extern int append_mode;
912 extern int sparse_files;
913 extern int keep_partial;
914 extern int checksum_seed;
917 extern int delay_updates;
918 extern mode_t orig_umask;
919 extern struct stats stats;
920 @@ -399,6 +401,9 @@ int recv_files(int f_in, char *local_name)
922 rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used);
924 + if (use_db && !always_checksum)
928 delayed_bits = bitbag_create(cur_flist->used + 1);
930 diff --git a/support/dbupdate b/support/dbupdate
933 +++ b/support/dbupdate
940 +use Cwd qw(abs_path cwd);
944 +my $MOUNT_FILE = '/etc/mtab';
946 +&Getopt::Long::Configure('bundling');
947 +&usage if !&GetOptions(
948 + 'db=s' => \( my $db_config ),
949 + 'mounts|m' => \( my $update_mounts ),
950 + 'recurse|r' => \( my $recurse_opt ),
951 + 'check|c' => \( my $check_opt ),
952 + 'verbose|v+' => \( my $verbosity = 0 ),
953 + 'help|h' => \( my $help_opt ),
955 +&usage if $help_opt || !defined $db_config;
958 +open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
962 + my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
963 + $config{$key} = $val;
967 +die "You must define at least dbtype and dbname in $db_config\n"
968 + unless defined $config{'dbtype'} && defined $config{'dbname'};
970 +my $thishost = $config{'thishost'} || 'localhost';
972 +my $connect = 'DBI:' . $config{'dbtype'} . ':database=' . $config{'dbname'};
973 +$connect =~ s/:database=/:dbname=/ if $config{'dbtype'} eq 'SQLite';
974 +$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
975 +$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
977 +my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
978 + or die "DB connection failed\n";
981 + $dbh->disconnect if defined $dbh;
984 +my $sel_disk_H = $dbh->prepare("
985 + SELECT disk_id, devno, mounted, comment
988 + ") or die $dbh->errstr;
990 +my $ins_disk_H = $dbh->prepare("
992 + (devno, host, mounted, comment)
994 + ") or die $dbh->errstr;
996 +my $up_disk_H = $dbh->prepare("
1000 + ") or die $dbh->errstr;
1002 +my $row_id = $config{'dbtype'} eq 'SQLite' ? 'ROWID' : 'ID';
1003 +my $sel_lastid_H = $dbh->prepare("
1004 + SELECT LAST_INSERT_$row_id()
1005 + ") or die $dbh->errstr;
1007 +my $sel_sum_H = $dbh->prepare("
1008 + SELECT sum_type, checksum
1010 + WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
1011 + ") or die $dbh->errstr;
1013 +my $rep_sum_H = $dbh->prepare("
1014 + REPLACE INTO inode_map
1015 + (disk_id, ino, size, mtime, ctime, sum_type, checksum)
1016 + VALUES(?, ?, ?, ?, ?, ?, ?)
1017 + ") or die $dbh->errstr;
1020 +if ($update_mounts) {
1021 + open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
1023 + my($devname, $mnt) = (split)[0,1];
1024 + next unless $devname =~ m#^/dev#;
1025 + my($devno) = (stat($mnt))[0];
1026 + if (!defined $devno) {
1027 + warn "Unable to stat $mnt: $!\n";
1030 + $mounts{$devno} = "$devname on $mnt";
1036 +$sel_disk_H->execute($thishost);
1037 +while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
1038 + if ($update_mounts) {
1039 + if (defined $mounts{$devno}) {
1040 + if ($comment ne $mounts{$devno}) {
1042 + $up_disk_H->execute(0, $disk_id);
1047 + $up_disk_H->execute(1, $disk_id);
1051 + $up_disk_H->execute(0, $disk_id);
1056 + next unless $mounted;
1058 + $disk_id{$devno} = $disk_id;
1060 +$sel_disk_H->finish;
1062 +if ($update_mounts) {
1063 + while (my($devno, $comment) = each %mounts) {
1064 + next if $disk_id{$devno};
1065 + $ins_disk_H->execute($devno, $thishost, 1, $comment);
1066 + $sel_lastid_H->execute;
1067 + ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
1068 + $sel_lastid_H->finish;
1072 +my $start_dir = cwd();
1075 +@dirs = '.' unless @dirs;
1077 + $_ = abs_path($_);
1084 +my $md4 = Digest::MD4->new;
1085 +my $md5 = Digest::MD5->new;
1088 + my $dir = shift @dirs;
1090 + if (!chdir($dir)) {
1091 + warn "Unable to chdir to $dir: $!\n";
1094 + if (!opendir(DP, '.')) {
1095 + warn "Unable to opendir $dir: $!\n";
1099 + my $reldir = $dir;
1100 + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
1101 + print "$reldir ... \n" if $verbosity;
1104 + while (defined(my $fn = readdir(DP))) {
1105 + next if $fn =~ /^\.\.?$/ || -l $fn;
1107 + push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
1112 + my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
1113 + my $disk_id = $disk_id{$dev} or next;
1114 + $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
1115 + my($sum4, $dbsum4, $sum5, $dbsum5);
1117 + while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
1118 + if ($sum_type == 4) {
1119 + $dbsum4 = $checksum;
1121 + } elsif ($sum_type == 5) {
1122 + $dbsum5 = $checksum;
1126 + $sel_sum_H->finish;
1128 + next if !$check_opt && $dbsumcnt == 2;
1130 + if (!$check_opt || $dbsumcnt || $verbosity > 2) {
1131 + if (!open(IN, $fn)) {
1132 + print STDERR "Unable to read $fn: $!\n";
1137 + while (sysread(IN, $_, 64*1024)) {
1141 + $sum4 = $md4->digest;
1142 + $sum5 = $md5->digest;
1143 + print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
1144 + print " $fn" if $verbosity > 1;
1145 + my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
1146 + last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
1151 + sysseek(IN, 0, 0);
1152 + print " REREADING\n" if $verbosity > 1;
1156 + } elsif ($verbosity > 1) {
1162 + if ($dbsumcnt == 0) {
1163 + $dif = ' --MISSING--';
1166 + if (!defined $dbsum4) {
1167 + $dif .= ' -NO-MD4-';
1168 + } elsif ($sum4 ne $dbsum4) {
1169 + $dif .= ' -MD4-CHANGED-';
1171 + if (!defined $dbsum5) {
1172 + $dif .= ' ---NO-MD5---';
1173 + } elsif ($sum5 ne $dbsum5) {
1174 + $dif .= ' -MD5-CHANGED-';
1177 + print " ====OK====\n" if $verbosity > 1;
1180 + $dif =~ s/MD4-CHANGED MD5-//;
1182 + if ($verbosity < 2) {
1183 + print $verbosity ? ' ' : "$reldir/";
1189 + print "\n" if $verbosity > 1;
1190 + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
1191 + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
1197 + unshift(@dirs, sort @subdirs) if $recurse_opt;
1205 +Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
1208 + --db=FILE Specify the config FILE to read for the DB info.
1209 + -m, --mounts Update mount info.
1210 + -r, --recurse Scan files in subdirectories too.
1211 + -c, --check Check if the checksums are right (doesn't update).
1212 + -v, --verbose Mention what we're doing. Repeat for more info.
1213 + -h, --help Display this help message.