Fix alignment issue on 64-bit. Solution from Steve Ortiz.
[rsync/rsync-patches.git] / db.diff
CommitLineData
cbdf862c
WD
1Added some DB-access routines to help rsync keep extra filesystem info
2about the files it is dealing with. This adds both the --db=CONFIG_FILE
3option and the "db config" daemon parameter.
4
5For the moment this only adds checksum caching when the --checksum option
bc3fcf1d 6is used. Future improvements may include:
cbdf862c
WD
7
8 - Updating of MD5 checksums when transferring any file, even w/o -c.
bc3fcf1d
WD
9 We should be able to extend this to work for MD4 checksums too if we
10 make the sender force checksum_seed to 0 when using a DB and having
11 the receiving side check to see if it got a 0 checksum_seed. (We
12 probably don't want to compute 2 MD4 checksums for the case where
13 the checksum_seed is non-zero.)
cbdf862c
WD
14
15 - Caching of path info that allows for the finding of files to use for
16 moving/linking/copying/alternate-basis-use.
17
18 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
19
20To use this patch, run these commands for a successful build:
21
cbdf862c
WD
22 patch -p1 <patches/db.diff
23 ./configure (optional if already run)
24 make
25
c1ff70aa 26based-on: a01e3b490eb36ccf9e704840e1b6683dab867550
cbdf862c
WD
27diff --git a/Makefile.in b/Makefile.in
28--- a/Makefile.in
29+++ b/Makefile.in
fc557362 30@@ -36,7 +36,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
cbdf862c 31 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
fc557362 32 util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
cbdf862c
WD
33 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
34- fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
35+ fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
36 OBJS3=progress.o pipe.o
37 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
38 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
39diff --git a/checksum.c b/checksum.c
40--- a/checksum.c
41+++ b/checksum.c
42@@ -23,6 +23,7 @@
43
44 extern int checksum_seed;
45 extern int protocol_version;
46+extern int use_db;
47
fc557362
WD
48 /*
49 a simple 32 bit checksum that can be upadted from either end
50@@ -98,10 +99,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
cbdf862c
WD
51 }
52 }
53
54-void file_checksum(char *fname, char *sum, OFF_T size)
55+void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
56 {
57 struct map_struct *buf;
58- OFF_T i, len = size;
59+ OFF_T i, len = st_p->st_size;
60 md_context m;
61 int32 remainder;
62 int fd;
fc557362 63@@ -112,7 +113,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
cbdf862c
WD
64 if (fd == -1)
65 return;
66
67- buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
68+ buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
69
70 if (protocol_version >= 30) {
71 md5_begin(&m);
fc557362 72@@ -146,6 +147,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
cbdf862c
WD
73 mdfour_result(&m, (uchar *)sum);
74 }
75
76+ if (use_db)
77+ db_set_checksum(fname, st_p, sum);
78+
79 close(fd);
80 unmap_file(buf);
81 }
82diff --git a/cleanup.c b/cleanup.c
83--- a/cleanup.c
84+++ b/cleanup.c
c1ff70aa
WD
85@@ -26,6 +26,7 @@ extern int am_server;
86 extern int am_daemon;
87 extern int am_receiver;
cbdf862c 88 extern int io_error;
fc557362 89+extern int use_db;
cbdf862c
WD
90 extern int keep_partial;
91 extern int got_xfer_error;
5214a41b 92 extern int protocol_version;
c1ff70aa 93@@ -140,6 +141,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
cbdf862c
WD
94 /* FALLTHROUGH */
95 #include "case_N.h"
96
97+ if (use_db)
98+ db_disconnect();
99+
100+ /* FALLTHROUGH */
101+#include "case_N.h"
102+
103 if (cleanup_child_pid != -1) {
104 int status;
105 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
106diff --git a/clientserver.c b/clientserver.c
107--- a/clientserver.c
108+++ b/clientserver.c
bc3fcf1d 109@@ -42,13 +42,16 @@ extern int numeric_ids;
cbdf862c
WD
110 extern int filesfrom_fd;
111 extern int remote_protocol;
112 extern int protocol_version;
113+extern int always_checksum;
114 extern int io_timeout;
115 extern int no_detach;
bc3fcf1d 116+extern int use_db;
cbdf862c 117 extern int write_batch;
bc3fcf1d 118 extern int default_af_hint;
cbdf862c
WD
119 extern int logfile_format_has_i;
120 extern int logfile_format_has_o_or_i;
121 extern mode_t orig_umask;
122+extern char *db_config;
123 extern char *bind_address;
cbdf862c 124 extern char *config_file;
fc557362 125 extern char *logfile_format;
5214a41b 126@@ -667,6 +670,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
cbdf862c 127
bc3fcf1d
WD
128 log_init(1);
129
130+ if (*lp_db_config(i))
131+ db_read_config(FLOG, lp_db_config(i));
cbdf862c 132+
bc3fcf1d
WD
133 #ifdef HAVE_PUTENV
134 if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
72e5645e 135 int status;
5214a41b 136@@ -862,6 +868,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
cbdf862c 137
bc3fcf1d
WD
138 am_server = 1; /* Don't let someone try to be tricky. */
139 quiet = 0;
140+ db_config = NULL;
141+ if (!always_checksum)
142+ use_db = 0;
143+
144 if (lp_ignore_errors(module_id))
145 ignore_errors = 1;
146 if (write_batch < 0)
cbdf862c
WD
147diff --git a/configure.in b/configure.in
148--- a/configure.in
149+++ b/configure.in
c1ff70aa 150@@ -337,7 +337,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
bc3fcf1d
WD
151 sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
152 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
153 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
154- popt.h popt/popt.h)
155+ popt.h popt/popt.h mysql/mysql.h sqlite3.h)
156 AC_HEADER_MAJOR
157
158 AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[
c1ff70aa 159@@ -1019,6 +1019,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
cbdf862c
WD
160 fi
161 fi
162
bc3fcf1d
WD
163+AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
164+if test x$MYSQL_CONFIG = x1; then
165+ AC_MSG_CHECKING(for mysql version >= 4)
166+ mysql_version=`mysql_config --version`
167+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
168+ if test $mysql_major_version -lt 4; then
169+ AC_MSG_RESULT(no.. skipping MySQL)
170+ else
171+ AC_MSG_RESULT(yes)
172+
173+ MYSQL_CFLAGS=`mysql_config --cflags`
174+ MYSQL_LIBS=`mysql_config --libs`
175+
176+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
177+ LIBS="$MYSQL_LIBS $LIBS"
178+
179+ AC_CHECK_LIB(mysqlclient, mysql_init)
180+ fi
181+fi
182+
183+AC_CHECK_LIB(sqlite3, sqlite3_open)
184+AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
cbdf862c
WD
185+
186 case "$CC" in
187 ' checker'*|checker*)
188 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
189diff --git a/db.c b/db.c
190new file mode 100644
191--- /dev/null
192+++ b/db.c
72e5645e 193@@ -0,0 +1,567 @@
cbdf862c
WD
194+/*
195+ * Routines to access extended file info via DB.
196+ *
197+ * Copyright (C) 2008 Wayne Davison
198+ *
199+ * This program is free software; you can redistribute it and/or modify
200+ * it under the terms of the GNU General Public License as published by
201+ * the Free Software Foundation; either version 3 of the License, or
202+ * (at your option) any later version.
203+ *
204+ * This program is distributed in the hope that it will be useful,
205+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
206+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
207+ * GNU General Public License for more details.
208+ *
209+ * You should have received a copy of the GNU General Public License along
210+ * with this program; if not, visit the http://fsf.org website.
211+ */
212+
213+#include "rsync.h"
214+#include "ifuncs.h"
72e5645e 215+#include "itypes.h"
cbdf862c 216+
bc3fcf1d 217+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
cbdf862c 218+#define USE_MYSQL
cbdf862c
WD
219+#include <mysql/mysql.h>
220+#include <mysql/errmsg.h>
221+#endif
bc3fcf1d
WD
222+
223+#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
224+#define USE_SQLITE
cbdf862c 225+#include <sqlite3.h>
bc3fcf1d
WD
226+#ifndef HAVE_SQLITE3_OPEN_V2
227+#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
228+ sqlite3_open(dbname, dbhptr)
229+#endif
230+#ifndef HAVE_SQLITE3_PREPARE_V2
231+#define sqlite3_prepare_v2 sqlite3_prepare
232+#endif
cbdf862c
WD
233+#endif
234+
235+extern int protocol_version;
236+extern int checksum_len;
237+
238+#define DB_TYPE_NONE 0
239+#define DB_TYPE_MYSQL 1
240+#define DB_TYPE_SQLITE 2
241+
242+int use_db = DB_TYPE_NONE;
243+
244+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
245+static unsigned int dbport = 0;
246+
247+static union {
248+#ifdef USE_MYSQL
249+ MYSQL *mysql;
250+#endif
251+#ifdef USE_SQLITE
252+ sqlite3 *sqlite;
253+#endif
254+ void *all;
255+} dbh;
256+
257+#define SEL_DEV 0
258+#define SEL_SUM 1
259+#define REP_SUM 2
260+#define MAX_PREP_CNT 3
261+
262+static union {
263+#ifdef USE_MYSQL
264+ MYSQL_STMT *mysql;
265+#endif
266+#ifdef USE_SQLITE
267+ sqlite3_stmt *sqlite;
268+#endif
269+ void *all;
270+} statements[MAX_PREP_CNT];
271+
272+static int md_num;
273+static enum logcode log_code;
274+
bc3fcf1d 275+#ifdef USE_MYSQL
cbdf862c
WD
276+static unsigned int bind_disk_id;
277+static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
bc3fcf1d
WD
278+static char bind_sum[MAX_DIGEST_LEN];
279+#endif
280+static char bind_thishost[256];
cbdf862c
WD
281+static int bind_thishost_len;
282+
283+static unsigned int prior_disk_id = 0;
284+static unsigned long long prior_devno = 0;
285+
286+int db_read_config(enum logcode code, const char *config_file)
287+{
288+ char buf[2048], *cp;
289+ FILE *fp;
290+ int lineno = 0;
291+
292+ log_code = code;
293+
294+ bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
295+
296+ if (!(fp = fopen(config_file, "r"))) {
297+ rsyserr(log_code, errno, "unable to open %s", config_file);
298+ return 0;
299+ }
300+ while (fgets(buf, sizeof buf, fp)) {
301+ lineno++;
302+ if ((cp = strchr(buf, '#')) == NULL
303+ && (cp = strchr(buf, '\r')) == NULL
304+ && (cp = strchr(buf, '\n')) == NULL)
305+ cp = buf + strlen(buf);
306+ while (cp != buf && isSpace(cp-1)) cp--;
307+ *cp = '\0';
308+
309+ if (!*buf)
310+ continue;
311+
312+ if (!(cp = strchr(buf, ':')))
313+ goto invalid_line;
314+ *cp++ = '\0';
315+
316+ while (isSpace(cp)) cp++;
317+ if (strcasecmp(buf, "dbhost") == 0)
318+ dbhost = strdup(cp);
319+ else if (strcasecmp(buf, "dbuser") == 0)
320+ dbuser = strdup(cp);
321+ else if (strcasecmp(buf, "dbpass") == 0)
322+ dbpass = strdup(cp);
323+ else if (strcasecmp(buf, "dbname") == 0)
324+ dbname = strdup(cp);
325+ else if (strcasecmp(buf, "dbport") == 0)
326+ dbport = atoi(cp);
327+ else if (strcasecmp(buf, "thishost") == 0)
328+ bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
329+ else if (strcasecmp(buf, "dbtype") == 0) {
330+#ifdef USE_MYSQL
331+ if (strcasecmp(cp, "mysql") == 0) {
332+ use_db = DB_TYPE_MYSQL;
333+ continue;
334+ }
335+#endif
336+#ifdef USE_SQLITE
337+ if (strcasecmp(cp, "sqlite") == 0) {
338+ use_db = DB_TYPE_SQLITE;
339+ continue;
340+ }
341+#endif
342+ rprintf(log_code,
343+ "Unsupported dbtype on line #%d in %s.\n",
344+ lineno, config_file);
345+ use_db = DB_TYPE_NONE;
346+ return 0;
347+ } else {
348+ invalid_line:
349+ rprintf(log_code, "Invalid line #%d in %s\n",
350+ lineno, config_file);
351+ use_db = DB_TYPE_NONE;
352+ return 0;
353+ }
354+ }
355+ fclose(fp);
356+
357+ if (bind_thishost_len >= (int)sizeof bind_thishost)
358+ bind_thishost_len = sizeof bind_thishost - 1;
359+
360+ if (!use_db || !dbname) {
361+ rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
362+ use_db = DB_TYPE_NONE;
363+ return 0;
364+ }
365+
366+ md_num = protocol_version >= 30 ? 5 : 4;
367+
368+ return 1;
369+}
370+
371+#ifdef USE_MYSQL
372+static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
373+{
374+ va_list ap;
375+ char *query;
376+ int qlen, param_cnt;
377+ MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
378+
379+ if (stmt == NULL)
380+ out_of_memory("prepare_mysql");
381+
382+ va_start(ap, fmt);
383+ qlen = vasprintf(&query, fmt, ap);
384+ va_end(ap);
385+ if (qlen < 0)
386+ out_of_memory("prepare_mysql");
387+
388+ if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
389+ rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
390+ return NULL;
391+ }
392+ free(query);
393+
394+ if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
395+ rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
396+ param_cnt, bind_cnt);
397+ return NULL;
398+ }
399+ if (bind_cnt)
400+ mysql_stmt_bind_param(stmt, binds);
401+
402+ return stmt;
403+}
404+#endif
405+
406+#ifdef USE_MYSQL
407+static int db_connect_mysql(void)
408+{
409+ MYSQL_BIND binds[10];
410+
411+ if (!(dbh.mysql = mysql_init(NULL)))
412+ out_of_memory("db_read_config");
413+
414+ if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
415+ return 0;
416+
417+ memset(binds, 0, sizeof binds);
418+ binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
419+ binds[0].buffer = &bind_devno;
420+ binds[1].buffer_type = MYSQL_TYPE_STRING;
421+ binds[1].buffer = &bind_thishost;
422+ binds[1].buffer_length = bind_thishost_len;
423+ statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
424+ "SELECT disk_id"
425+ " FROM disk"
426+ " WHERE devno = ? AND host = ? AND mounted = 1");
427+ if (!statements[SEL_DEV].mysql)
428+ return 0;
429+
430+ memset(binds, 0, sizeof binds);
431+ binds[0].buffer_type = MYSQL_TYPE_LONG;
432+ binds[0].buffer = &bind_disk_id;
433+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
434+ binds[1].buffer = &bind_ino;
435+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
436+ binds[2].buffer = &bind_size;
437+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
438+ binds[3].buffer = &bind_mtime;
439+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
440+ binds[4].buffer = &bind_ctime;
441+ statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
442+ "SELECT checksum"
443+ " FROM inode_map"
444+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
445+ " AND size = ? AND mtime = ? AND ctime = ?",
446+ md_num);
447+ if (!statements[SEL_SUM].mysql)
448+ return 0;
449+
450+ memset(binds, 0, sizeof binds);
451+ binds[0].buffer_type = MYSQL_TYPE_LONG;
452+ binds[0].buffer = &bind_disk_id;
453+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
454+ binds[1].buffer = &bind_ino;
455+ binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
456+ binds[2].buffer = binds[6].buffer = &bind_size;
457+ binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
458+ binds[3].buffer = binds[7].buffer = &bind_mtime;
459+ binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
460+ binds[4].buffer = binds[8].buffer = &bind_ctime;
461+ binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
462+ binds[5].buffer = binds[9].buffer = &bind_sum;
463+ binds[5].buffer_length = binds[9].buffer_length = checksum_len;
464+ statements[REP_SUM].mysql = prepare_mysql(binds, 10,
465+ "INSERT INTO inode_map"
466+ " SET disk_id = ?, ino = ?, sum_type = %d,"
467+ " size = ?, mtime = ?, ctime = ?, checksum = ?"
468+ " ON DUPLICATE KEY"
469+ " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
470+ md_num, md_num);
471+ if (!statements[REP_SUM].mysql)
472+ return 0;
473+
474+ return 1;
475+}
476+#endif
477+
478+#ifdef USE_SQLITE
479+static int db_connect_sqlite(void)
480+{
481+ char *sql;
482+
cbdf862c
WD
483+ if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
484+ return 0;
cbdf862c
WD
485+
486+ sql = "SELECT disk_id"
487+ " FROM disk"
488+ " WHERE devno = ? AND host = ? AND mounted = 1";
489+ if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
490+ return 0;
491+
492+ if (asprintf(&sql,
493+ "SELECT checksum"
494+ " FROM inode_map"
495+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
496+ " AND size = ? AND mtime = ? AND ctime = ?",
497+ md_num) < 0
498+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
499+ return 0;
500+ free(sql);
501+
502+ if (asprintf(&sql,
503+ "INSERT OR REPLACE INTO inode_map"
504+ " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
505+ " VALUES(?, ?, %d, ?, ?, ?, ?)",
506+ md_num) < 0
507+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
508+ return 0;
509+ free(sql);
510+
511+ return 1;
512+}
513+#endif
514+
515+int db_connect(void)
516+{
517+ switch (use_db) {
518+#ifdef USE_MYSQL
519+ case DB_TYPE_MYSQL:
520+ if (db_connect_mysql())
521+ return 1;
522+ break;
523+#endif
524+#ifdef USE_SQLITE
525+ case DB_TYPE_SQLITE:
526+ if (db_connect_sqlite())
527+ return 1;
528+ break;
529+#endif
530+ }
531+
532+ rprintf(log_code, "Unable to connect to DB\n");
533+ db_disconnect();
534+ use_db = DB_TYPE_NONE;
535+
536+ return 0;
537+}
538+
539+void db_disconnect(void)
540+{
541+ int ndx;
542+
543+ if (!dbh.all)
544+ return;
545+
546+ for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
547+ if (statements[ndx].all) {
548+ switch (use_db) {
549+#ifdef USE_MYSQL
550+ case DB_TYPE_MYSQL:
551+ mysql_stmt_close(statements[ndx].mysql);
552+ break;
553+#endif
554+#ifdef USE_SQLITE
555+ case DB_TYPE_SQLITE:
556+ sqlite3_finalize(statements[ndx].sqlite);
557+ break;
558+#endif
559+ }
560+ statements[ndx].all = NULL;
561+ }
562+ }
563+
564+ switch (use_db) {
565+#ifdef USE_MYSQL
566+ case DB_TYPE_MYSQL:
567+ mysql_close(dbh.mysql);
568+ break;
569+#endif
570+#ifdef USE_SQLITE
571+ case DB_TYPE_SQLITE:
572+ sqlite3_close(dbh.sqlite);
573+ break;
574+#endif
575+ }
576+
577+ dbh.all = NULL;
578+}
579+
bc3fcf1d 580+#ifdef USE_MYSQL
cbdf862c
WD
581+static MYSQL_STMT *exec_mysql(int ndx)
582+{
583+ MYSQL_STMT *stmt = statements[ndx].mysql;
584+ int rc;
585+
586+ if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
587+ db_disconnect();
588+ if (db_connect()) {
589+ stmt = statements[ndx].mysql;
590+ rc = mysql_stmt_execute(stmt);
591+ }
592+ }
593+ if (rc != 0) {
594+ rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
595+ return NULL;
596+ }
597+
598+ return stmt;
599+}
bc3fcf1d 600+#endif
cbdf862c 601+
bc3fcf1d 602+#ifdef USE_MYSQL
cbdf862c
WD
603+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
604+{
605+ unsigned long length[32];
606+ my_bool is_null[32], error[32];
607+ MYSQL_STMT *stmt;
608+ int i, rc;
609+
610+ if (bind_cnt > 32)
611+ exit_cleanup(RERR_UNSUPPORTED);
612+
613+ if ((stmt = exec_mysql(ndx)) == NULL)
614+ return 0;
615+
616+ for (i = 0; i < bind_cnt; i++) {
617+ binds[i].is_null = &is_null[i];
618+ binds[i].length = &length[i];
619+ binds[i].error = &error[i];
620+ }
621+ mysql_stmt_bind_result(stmt, binds);
622+
623+ if ((rc = mysql_stmt_fetch(stmt)) != 0) {
624+ if (rc != MYSQL_NO_DATA) {
625+ rprintf(log_code, "SELECT fetch failed: %s\n",
626+ mysql_stmt_error(stmt));
627+ }
628+ mysql_stmt_free_result(stmt);
629+ return 0;
630+ }
631+
632+ mysql_stmt_free_result(stmt);
633+
634+ return is_null[0] ? 0 : 1;
635+}
bc3fcf1d 636+#endif
cbdf862c
WD
637+
638+static void get_disk_id(unsigned long long devno)
639+{
640+ switch (use_db) {
641+#ifdef USE_MYSQL
642+ case DB_TYPE_MYSQL: {
643+ MYSQL_BIND binds[1];
644+
645+ bind_devno = devno; /* The one variable SEL_DEV input value. */
646+
647+ /* Bind where to put the output. */
648+ binds[0].buffer_type = MYSQL_TYPE_LONG;
649+ binds[0].buffer = &prior_disk_id;
650+ if (!fetch_mysql(binds, 1, SEL_DEV))
651+ prior_disk_id = 0;
652+ break;
653+ }
654+#endif
655+#ifdef USE_SQLITE
656+ case DB_TYPE_SQLITE: {
657+ sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
658+ sqlite3_bind_int64(stmt, 1, devno);
659+ sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
660+ if (sqlite3_step(stmt) == SQLITE_ROW)
661+ prior_disk_id = sqlite3_column_int(stmt, 0);
662+ else
663+ prior_disk_id = 0;
664+ sqlite3_reset(stmt);
665+ break;
666+ }
667+#endif
668+ }
669+
670+ prior_devno = devno;
671+}
672+
673+int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
674+{
675+ if (prior_devno != st_p->st_dev)
676+ get_disk_id(st_p->st_dev);
677+ if (prior_disk_id == 0)
678+ return 0;
679+
680+ switch (use_db) {
681+#ifdef USE_MYSQL
682+ case DB_TYPE_MYSQL: {
683+ MYSQL_BIND binds[1];
684+
685+ bind_disk_id = prior_disk_id;
686+ bind_ino = st_p->st_ino;
687+ bind_size = st_p->st_size;
688+ bind_mtime = st_p->st_mtime;
689+ bind_ctime = st_p->st_ctime;
690+
691+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
692+ binds[0].buffer = sum;
693+ binds[0].buffer_length = checksum_len;
694+ return fetch_mysql(binds, 1, SEL_SUM);
695+ }
696+#endif
697+#ifdef USE_SQLITE
698+ case DB_TYPE_SQLITE: {
699+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
700+ sqlite3_bind_int(stmt, 1, prior_disk_id);
701+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
702+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
703+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
704+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
705+ if (sqlite3_step(stmt) == SQLITE_ROW) {
706+ int len = sqlite3_column_bytes(stmt, 0);
707+ if (len > MAX_DIGEST_LEN)
708+ len = MAX_DIGEST_LEN;
709+ memcpy(sum, sqlite3_column_blob(stmt, 0), len);
710+ sqlite3_reset(stmt);
711+ return 1;
712+ }
713+ sqlite3_reset(stmt);
714+ return 0;
715+ }
716+#endif
717+ }
718+
719+ return 0;
720+}
721+
722+int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
723+{
724+ if (prior_devno != st_p->st_dev)
725+ get_disk_id(st_p->st_dev);
726+ if (prior_disk_id == 0)
727+ return 0;
728+
729+ switch (use_db) {
730+#ifdef USE_MYSQL
731+ case DB_TYPE_MYSQL: {
732+ bind_disk_id = prior_disk_id;
733+ bind_ino = st_p->st_ino;
734+ bind_size = st_p->st_size;
735+ bind_mtime = st_p->st_mtime;
736+ bind_ctime = st_p->st_ctime;
737+ memcpy(bind_sum, sum, checksum_len);
738+
739+ return exec_mysql(REP_SUM) != NULL;
740+ }
741+#endif
742+#ifdef USE_SQLITE
743+ case DB_TYPE_SQLITE: {
744+ int rc;
745+ sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
746+ sqlite3_bind_int(stmt, 1, prior_disk_id);
747+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
748+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
749+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
750+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
751+ sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
752+ rc = sqlite3_step(stmt);
753+ sqlite3_reset(stmt);
754+ return rc == SQLITE_DONE;
755+ }
756+#endif
757+ }
758+
759+ return 0;
760+}
761diff --git a/flist.c b/flist.c
762--- a/flist.c
763+++ b/flist.c
c1ff70aa 764@@ -54,6 +54,7 @@ extern int delete_during;
fc557362 765 extern int missing_args;
cbdf862c
WD
766 extern int uid_ndx;
767 extern int gid_ndx;
768+extern int use_db;
769 extern int eol_nulls;
770 extern int relative_paths;
771 extern int implied_dirs;
c1ff70aa 772@@ -1309,11 +1310,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
fc557362 773 extra_len += EXTRA_LEN;
cbdf862c
WD
774 #endif
775
fc557362 776- if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
cbdf862c 777- file_checksum(thisname, tmp_sum, st.st_size);
fc557362
WD
778- if (sender_keeps_checksum)
779- extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
780- }
781+ if (sender_keeps_checksum && S_ISREG(st.st_mode))
782+ extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
cbdf862c 783
fc557362
WD
784 #if EXTRA_ROUNDING > 0
785 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
c1ff70aa 786@@ -1396,8 +1394,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
fc557362
WD
787 return NULL;
788 }
789
790- if (sender_keeps_checksum && S_ISREG(st.st_mode))
791- memcpy(F_SUM(file), tmp_sum, checksum_len);
cbdf862c
WD
792+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
793+ if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
794+ file_checksum(thisname, &st, tmp_sum);
fc557362
WD
795+ if (sender_keeps_checksum)
796+ memcpy(F_SUM(file), tmp_sum, checksum_len);
cbdf862c 797+ }
fc557362
WD
798
799 if (unsort_ndx)
800 F_NDX(file) = stats.num_dirs;
c1ff70aa 801@@ -2075,6 +2077,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
cbdf862c
WD
802 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
803 int implied_dot_dir = 0;
804
805+ if (use_db)
806+ db_connect();
807+
808 rprintf(FLOG, "building file list\n");
809 if (show_filelist_p())
810 start_filelist_progress("building file list");
811diff --git a/generator.c b/generator.c
812--- a/generator.c
813+++ b/generator.c
fc557362 814@@ -60,6 +60,7 @@ extern int human_readable;
cbdf862c
WD
815 extern int ignore_existing;
816 extern int ignore_non_existing;
817 extern int inplace;
818+extern int use_db;
819 extern int append_mode;
820 extern int make_backups;
821 extern int csum_length;
c1ff70aa 822@@ -530,7 +531,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
cbdf862c
WD
823 of the file time to determine whether to sync */
824 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
825 char sum[MAX_DIGEST_LEN];
826- file_checksum(fn, sum, st->st_size);
827+ if (!use_db || !db_get_checksum(fn, st, sum))
828+ file_checksum(fn, st, sum);
829 return memcmp(sum, F_SUM(file), checksum_len) == 0;
830 }
831
c1ff70aa 832@@ -2075,6 +2077,9 @@ void generate_files(int f_out, const char *local_name)
cbdf862c
WD
833 : "enabled");
834 }
835
836+ if (use_db && always_checksum)
837+ db_connect();
838+
c1ff70aa
WD
839 dflt_perms = (ACCESSPERMS & ~orig_umask);
840
841 do {
cbdf862c
WD
842diff --git a/loadparm.c b/loadparm.c
843--- a/loadparm.c
844+++ b/loadparm.c
72e5645e 845@@ -108,6 +108,7 @@ typedef struct {
cbdf862c
WD
846 char *auth_users;
847 char *charset;
848 char *comment;
849+ char *db_config;
850 char *dont_compress;
851 char *exclude;
852 char *exclude_from;
72e5645e 853@@ -182,6 +183,7 @@ static const all_vars Defaults = {
cbdf862c
WD
854 /* auth_users; */ NULL,
855 /* charset; */ NULL,
856 /* comment; */ NULL,
857+ /* db_config; */ NULL,
858 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
859 /* exclude; */ NULL,
860 /* exclude_from; */ NULL,
72e5645e 861@@ -317,6 +319,7 @@ static struct parm_struct parm_table[] =
fc557362
WD
862 {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0},
863 {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0},
864 {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0},
865+ {"db config", P_STRING, P_LOCAL, &Vars.l.db_config, NULL,0},
866 {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0},
867 {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0},
868 {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0},
72e5645e 869@@ -447,6 +450,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
cbdf862c
WD
870 FN_LOCAL_STRING(lp_auth_users, auth_users)
871 FN_LOCAL_STRING(lp_charset, charset)
872 FN_LOCAL_STRING(lp_comment, comment)
873+FN_LOCAL_STRING(lp_db_config, db_config)
874 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
875 FN_LOCAL_STRING(lp_exclude, exclude)
876 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
877diff --git a/main.c b/main.c
878--- a/main.c
879+++ b/main.c
5214a41b 880@@ -51,6 +51,7 @@ extern int copy_unsafe_links;
cbdf862c
WD
881 extern int keep_dirlinks;
882 extern int preserve_hard_links;
883 extern int protocol_version;
884+extern int always_checksum;
885 extern int file_total;
886 extern int recurse;
887 extern int xfer_dirs;
c1ff70aa 888@@ -83,6 +84,7 @@ extern char *filesfrom_host;
fc557362 889 extern char *partial_dir;
cbdf862c 890 extern char *dest_option;
cbdf862c
WD
891 extern char *rsync_path;
892+extern char *db_config;
893 extern char *shell_cmd;
894 extern char *batch_name;
895 extern char *password_file;
c1ff70aa 896@@ -1609,6 +1611,9 @@ int main(int argc,char *argv[])
cbdf862c
WD
897 exit_cleanup(RERR_SYNTAX);
898 }
899
bc3fcf1d 900+ if (db_config && always_checksum)
cbdf862c
WD
901+ db_read_config(FERROR, db_config);
902+
903 if (am_server) {
904 set_nonblocking(STDIN_FILENO);
905 set_nonblocking(STDOUT_FILENO);
906diff --git a/options.c b/options.c
907--- a/options.c
908+++ b/options.c
909@@ -92,6 +92,7 @@ int use_qsort = 0;
910 char *files_from = NULL;
911 int filesfrom_fd = -1;
912 char *filesfrom_host = NULL;
913+char *db_config = NULL;
914 int eol_nulls = 0;
c1ff70aa 915 int protect_args = -1;
fc557362 916 int human_readable = 1;
72e5645e 917@@ -567,6 +568,7 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
918 char const *links = "no ";
919 char const *iconv = "no ";
920 char const *ipv6 = "no ";
921+ char const *db = "no ";
922 STRUCT_STAT *dumstat;
923
924 #if SUBPROTOCOL_VERSION != 0
72e5645e
WD
925@@ -600,6 +602,11 @@ static void print_rsync_version(enum logcode f)
926 #ifdef CAN_SET_SYMLINK_TIMES
bc3fcf1d
WD
927 symtimes = "";
928 #endif
929+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
930+ db = "";
931+#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
932+ db = "";
933+#endif
934
935 rprintf(f, "%s version %s protocol version %d%s\n",
936 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
72e5645e 937@@ -613,8 +620,8 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
938 (int)(sizeof (int64) * 8));
939 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
940 got_socketpair, hardlinks, links, ipv6, have_inplace);
941- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
942- have_inplace, acls, xattrs, iconv, symtimes);
943+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n",
944+ have_inplace, acls, xattrs, iconv, symtimes, db);
945
946 #ifdef MAINTAINER_MODE
947 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
72e5645e 948@@ -662,6 +669,7 @@ void usage(enum logcode F)
cbdf862c
WD
949 rprintf(F," -q, --quiet suppress non-error messages\n");
950 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
951 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
bc3fcf1d 952+ rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
cbdf862c
WD
953 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
954 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
955 rprintf(F," -r, --recursive recurse into directories\n");
72e5645e 956@@ -934,6 +942,7 @@ static struct poptOption long_options[] = {
cbdf862c
WD
957 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
958 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
959 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
960+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
961 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
962 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
963 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
964diff --git a/pipe.c b/pipe.c
965--- a/pipe.c
966+++ b/pipe.c
fc557362 967@@ -27,6 +27,9 @@ extern int am_server;
cbdf862c
WD
968 extern int blocking_io;
969 extern int filesfrom_fd;
fc557362 970 extern int munge_symlinks;
cbdf862c 971+extern int always_checksum;
cbdf862c
WD
972+extern int use_db;
973+extern char *db_config;
974 extern mode_t orig_umask;
975 extern char *logfile_name;
976 extern int remote_option_cnt;
fc557362 977@@ -143,6 +146,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
978 logfile_close();
979 }
980
981+ use_db = 0;
982+ db_config = NULL;
983+
984 if (remote_option_cnt) {
985 int rc = remote_option_cnt + 1;
986 const char **rv = remote_options;
fc557362 987@@ -150,6 +156,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
988 option_error();
989 exit_cleanup(RERR_SYNTAX);
990 }
bc3fcf1d 991+ if (db_config && always_checksum)
cbdf862c
WD
992+ db_read_config(FERROR, db_config);
993 }
994
995 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
bc3fcf1d
WD
996diff --git a/rsync.yo b/rsync.yo
997--- a/rsync.yo
998+++ b/rsync.yo
fc557362 999@@ -323,6 +323,7 @@ to the detailed description below for a complete description. verb(
bc3fcf1d
WD
1000 -q, --quiet suppress non-error messages
1001 --no-motd suppress daemon-mode MOTD (see caveat)
1002 -c, --checksum skip based on checksum, not mod-time & size
1003+ --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
1004 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
1005 --no-OPTION turn off an implied OPTION (e.g. --no-D)
1006 -r, --recursive recurse into directories
7170ca8d
WD
1007@@ -587,6 +588,47 @@ option's before-the-transfer "Does this file need to be updated?" check.
1008 For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
1009 MD5. For older protocols, the checksum used is MD4.
cbdf862c 1010
bc3fcf1d
WD
1011+dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
1012+that holds connection details for a database of checksum information.
1013+When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
1014+use cached checksum information from the DB, and will update it if it is
1015+missing.
1016+
1017+The currently supported DB choices are MySQL and SQLite. For example, a
1018+MySQL configuration might look like this:
1019+
1020+verb( dbtype: mysql
1021+ dbhost: 127.0.0.1
1022+ dbname: rsyncdb
1023+ dbuser: rsyncuser
1024+ dbpass: somepass
1025+ port: 3306
1026+ thishost: hostname )
1027+
1028+And a SQLite configuration might look like this:
1029+
1030+verb( dbtype: SQLite
1031+ dbname: /var/cache/rsync/sum.db )
1032+
1033+This option only affects one side of a transfer. See the
1034+bf(--remote-option) option for a way to specify the option for both
1035+sides of the transfer (with each side reading the config file from
1036+their local filesystem). For example:
1037+
1038+verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
1039+
1040+See the perl script "rsyncdb" in the support directory of the source code
1041+(which may also be installed in /usr/bin) for a way to create the tables,
1042+populate the mounted-disk information, check files against their checksums,
1043+and update both the MD4 and MD5 checksums for files at the same time (since
1044+an rsync copy will only update one or the other).
1045+
1046+You can use a single MySQL DB for all your hosts if you give each one
1047+their own "thishost" name and setup their device-mapping data. Or feel
1048+free to use separate databases, separate servers, etc. See the rsync
1049+daemon's "db config" parameter for how to configure a daemon to use a DB
1050+(since a client cannot control this parameter on a daemon).
1051+
1052 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
1053 way of saying you want recursion and want to preserve almost
1054 everything (with -H being a notable omission).
1055diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
1056--- a/rsyncd.conf.yo
1057+++ b/rsyncd.conf.yo
72e5645e 1058@@ -301,6 +301,18 @@ is daemon. This setting has no effect if the "log file" setting is a
bc3fcf1d
WD
1059 non-empty string (either set in the per-modules settings, or inherited
1060 from the global settings).
cbdf862c 1061
bc3fcf1d
WD
1062+dit(bf(db config)) This parameter specifies a config file to read that
1063+holds connection details for a database of checksum information.
1064+
1065+The config file will be read-in prior to any chroot restrictions, but
1066+the connection occurs from inside the chroot. This means that you
1067+should use a socket connection (e.g. 127.0.0.1 rather than localhost)
1068+for a MySQL config from inside a chroot. For SQLite, the DB file must
1069+be placed inside the chroot (though it can be placed outside the
1070+transfer dir if you configured an inside-chroot path).
1071+
1072+See the bf(--db=CONFIG_FILE) option for full details.
1073+
e66d6d51 1074 dit(bf(max verbosity)) This parameter allows you to control
bc3fcf1d
WD
1075 the maximum amount of verbose information that you'll allow the daemon to
1076 generate (since the information goes into the log file). The default is 1,
1077diff --git a/support/rsyncdb b/support/rsyncdb
cbdf862c
WD
1078new file mode 100755
1079--- /dev/null
bc3fcf1d 1080+++ b/support/rsyncdb
fc557362
WD
1081@@ -0,0 +1,331 @@
1082+#!/usr/bin/perl -w
cbdf862c
WD
1083+use strict;
1084+
1085+use DBI;
1086+use Getopt::Long;
1087+use Cwd qw(abs_path cwd);
1088+use Digest::MD4;
1089+use Digest::MD5;
1090+
1091+my $MOUNT_FILE = '/etc/mtab';
1092+
1093+&Getopt::Long::Configure('bundling');
1094+&usage if !&GetOptions(
1095+ 'db=s' => \( my $db_config ),
bc3fcf1d 1096+ 'init' => \( my $init_db ),
cbdf862c
WD
1097+ 'mounts|m' => \( my $update_mounts ),
1098+ 'recurse|r' => \( my $recurse_opt ),
1099+ 'check|c' => \( my $check_opt ),
1100+ 'verbose|v+' => \( my $verbosity = 0 ),
1101+ 'help|h' => \( my $help_opt ),
1102+);
1103+&usage if $help_opt || !defined $db_config;
1104+
1105+my %config;
1106+open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
1107+while (<IN>) {
1108+ s/[#\r\n].*//s;
1109+ next if /^$/;
1110+ my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
1111+ $config{$key} = $val;
1112+}
1113+close IN;
1114+
1115+die "You must define at least dbtype and dbname in $db_config\n"
1116+ unless defined $config{'dbtype'} && defined $config{'dbname'};
1117+
bc3fcf1d
WD
1118+my $sqlite = $config{'dbtype'} =~ /^sqlite$/i;
1119+
cbdf862c
WD
1120+my $thishost = $config{'thishost'} || 'localhost';
1121+
bc3fcf1d
WD
1122+my $connect = 'DBI:' . $config{'dbtype'} . ':';
1123+$connect .= 'dbname=' . $config{'dbname'} if $sqlite;
1124+$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db;
cbdf862c
WD
1125+$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
1126+$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
1127+
1128+my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
1129+ or die "DB connection failed\n";
1130+
1131+END {
1132+ $dbh->disconnect if defined $dbh;
1133+}
1134+
bc3fcf1d
WD
1135+if ($init_db) {
1136+ my $unsigned = $sqlite ? '' : 'unsigned';
1137+ my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT';
1138+ my $dbname = $config{'dbname'};
1139+
1140+ if (!$sqlite) {
1141+ $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`");
1142+ $dbh->do("USE `$dbname`");
1143+ }
1144+
1145+ print "Dropping old tables (if they exist) ...\n" if $verbosity;
1146+ $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr;
1147+ $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr;
1148+
1149+ print "Creating empty tables ...\n" if $verbosity;
1150+ $dbh->do("
1151+ CREATE TABLE disk (
1152+ disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment,
1153+ devno bigint $unsigned NOT NULL,
1154+ host varchar(256) NOT NULL default 'localhost',
1155+ mounted tinyint NOT NULL default '1',
1156+ comment varchar(256) default NULL
1157+ )") or die $dbh->errstr;
1158+
1159+ $dbh->do("
1160+ CREATE TABLE inode_map (
1161+ disk_id integer $unsigned NOT NULL,
1162+ ino bigint $unsigned NOT NULL,
1163+ size bigint $unsigned NOT NULL,
1164+ mtime bigint NOT NULL,
1165+ ctime bigint NOT NULL,
1166+ sum_type tinyint NOT NULL default '0',
1167+ checksum binary(16) NOT NULL,
1168+ PRIMARY KEY (disk_id,ino,sum_type)
1169+ )") or die $dbh->errstr;
1170+
1171+ exit unless $update_mounts;
1172+}
1173+
cbdf862c
WD
1174+my $sel_disk_H = $dbh->prepare("
1175+ SELECT disk_id, devno, mounted, comment
1176+ FROM disk
1177+ WHERE host = ?
1178+ ") or die $dbh->errstr;
1179+
1180+my $ins_disk_H = $dbh->prepare("
1181+ INSERT INTO disk
1182+ (devno, host, mounted, comment)
1183+ VALUES(?, ?, ?, ?)
1184+ ") or die $dbh->errstr;
1185+
1186+my $up_disk_H = $dbh->prepare("
1187+ UPDATE disk
1188+ SET mounted = ?
1189+ WHERE disk_id = ?
1190+ ") or die $dbh->errstr;
1191+
bc3fcf1d 1192+my $row_id = $sqlite ? 'ROWID' : 'ID';
cbdf862c
WD
1193+my $sel_lastid_H = $dbh->prepare("
1194+ SELECT LAST_INSERT_$row_id()
1195+ ") or die $dbh->errstr;
1196+
1197+my $sel_sum_H = $dbh->prepare("
1198+ SELECT sum_type, checksum
1199+ FROM inode_map
1200+ WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
1201+ ") or die $dbh->errstr;
1202+
1203+my $rep_sum_H = $dbh->prepare("
1204+ REPLACE INTO inode_map
1205+ (disk_id, ino, size, mtime, ctime, sum_type, checksum)
1206+ VALUES(?, ?, ?, ?, ?, ?, ?)
1207+ ") or die $dbh->errstr;
1208+
1209+my %mounts;
1210+if ($update_mounts) {
1211+ open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
1212+ while (<IN>) {
1213+ my($devname, $mnt) = (split)[0,1];
1214+ next unless $devname =~ m#^/dev#;
1215+ my($devno) = (stat($mnt))[0];
1216+ if (!defined $devno) {
1217+ warn "Unable to stat $mnt: $!\n";
1218+ next;
1219+ }
1220+ $mounts{$devno} = "$devname on $mnt";
1221+ }
1222+ close IN;
1223+}
1224+
1225+my %disk_id;
1226+$sel_disk_H->execute($thishost);
1227+while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
1228+ if ($update_mounts) {
1229+ if (defined $mounts{$devno}) {
1230+ if ($comment ne $mounts{$devno}) {
1231+ if ($mounted) {
bc3fcf1d 1232+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1233+ $up_disk_H->execute(0, $disk_id);
1234+ }
1235+ next;
1236+ }
1237+ if (!$mounted) {
bc3fcf1d 1238+ print "Mounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1239+ $up_disk_H->execute(1, $disk_id);
1240+ }
1241+ } else {
1242+ if ($mounted) {
bc3fcf1d 1243+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1244+ $up_disk_H->execute(0, $disk_id);
1245+ }
1246+ next;
1247+ }
1248+ } else {
1249+ next unless $mounted;
1250+ }
1251+ $disk_id{$devno} = $disk_id;
1252+}
1253+$sel_disk_H->finish;
1254+
1255+if ($update_mounts) {
1256+ while (my($devno, $comment) = each %mounts) {
1257+ next if $disk_id{$devno};
bc3fcf1d 1258+ print "Adding $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1259+ $ins_disk_H->execute($devno, $thishost, 1, $comment);
1260+ $sel_lastid_H->execute;
1261+ ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
1262+ $sel_lastid_H->finish;
1263+ }
bc3fcf1d 1264+ exit;
cbdf862c
WD
1265+}
1266+
1267+my $start_dir = cwd();
1268+
1269+my @dirs = @ARGV;
1270+@dirs = '.' unless @dirs;
1271+foreach (@dirs) {
1272+ $_ = abs_path($_);
1273+}
1274+
1275+$| = 1;
1276+
1277+my $exit_code = 0;
1278+
1279+my $md4 = Digest::MD4->new;
1280+my $md5 = Digest::MD5->new;
1281+
1282+while (@dirs) {
1283+ my $dir = shift @dirs;
1284+
1285+ if (!chdir($dir)) {
1286+ warn "Unable to chdir to $dir: $!\n";
1287+ next;
1288+ }
1289+ if (!opendir(DP, '.')) {
1290+ warn "Unable to opendir $dir: $!\n";
1291+ next;
1292+ }
1293+
1294+ my $reldir = $dir;
1295+ $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
1296+ print "$reldir ... \n" if $verbosity;
1297+
1298+ my @subdirs;
1299+ while (defined(my $fn = readdir(DP))) {
1300+ next if $fn =~ /^\.\.?$/ || -l $fn;
1301+ if (-d _) {
1302+ push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
1303+ next;
1304+ }
1305+ next unless -f _;
1306+
1307+ my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
1308+ my $disk_id = $disk_id{$dev} or next;
1309+ $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
1310+ my($sum4, $dbsum4, $sum5, $dbsum5);
1311+ my $dbsumcnt = 0;
1312+ while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
1313+ if ($sum_type == 4) {
1314+ $dbsum4 = $checksum;
1315+ $dbsumcnt++;
1316+ } elsif ($sum_type == 5) {
1317+ $dbsum5 = $checksum;
1318+ $dbsumcnt++;
1319+ }
1320+ }
1321+ $sel_sum_H->finish;
1322+
1323+ next if !$check_opt && $dbsumcnt == 2;
1324+
1325+ if (!$check_opt || $dbsumcnt || $verbosity > 2) {
1326+ if (!open(IN, $fn)) {
1327+ print STDERR "Unable to read $fn: $!\n";
1328+ next;
1329+ }
1330+
1331+ while (1) {
1332+ while (sysread(IN, $_, 64*1024)) {
1333+ $md4->add($_);
1334+ $md5->add($_);
1335+ }
1336+ $sum4 = $md4->digest;
1337+ $sum5 = $md5->digest;
1338+ print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
1339+ print " $fn" if $verbosity > 1;
1340+ my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
1341+ last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
1342+ $ino = $ino2;
1343+ $size = $size2;
1344+ $mtime = $mtime2;
1345+ $ctime = $ctime2;
1346+ sysseek(IN, 0, 0);
1347+ print " REREADING\n" if $verbosity > 1;
1348+ }
1349+
1350+ close IN;
1351+ } elsif ($verbosity > 1) {
1352+ print "_$fn";
1353+ }
1354+
1355+ if ($check_opt) {
1356+ my $dif;
1357+ if ($dbsumcnt == 0) {
1358+ $dif = ' --MISSING--';
1359+ } else {
1360+ $dif = '';
1361+ if (!defined $dbsum4) {
1362+ $dif .= ' -NO-MD4-';
1363+ } elsif ($sum4 ne $dbsum4) {
1364+ $dif .= ' -MD4-CHANGED-';
1365+ }
1366+ if (!defined $dbsum5) {
1367+ $dif .= ' ---NO-MD5---';
1368+ } elsif ($sum5 ne $dbsum5) {
1369+ $dif .= ' -MD5-CHANGED-';
1370+ }
1371+ if ($dif eq '') {
1372+ print " ====OK====\n" if $verbosity > 1;
1373+ next;
1374+ }
1375+ $dif =~ s/MD4-CHANGED MD5-//;
1376+ }
1377+ if ($verbosity < 2) {
1378+ print $verbosity ? ' ' : "$reldir/";
1379+ print $fn;
1380+ }
1381+ print $dif, "\n";
1382+ $exit_code = 1;
1383+ } else {
1384+ print "\n" if $verbosity > 1;
1385+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
1386+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
1387+ }
1388+ }
1389+
1390+ closedir DP;
1391+
1392+ unshift(@dirs, sort @subdirs) if $recurse_opt;
1393+}
1394+
1395+exit $exit_code;
1396+
1397+sub usage
1398+{
1399+ die <<EOT;
1400+Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
1401+
1402+Options:
1403+ --db=FILE Specify the config FILE to read for the DB info.
bc3fcf1d
WD
1404+ --init Create (recreate) needed tables (making them empty).
1405+ No DIR scanning, but can be combined with --mounts.
1406+ -m, --mounts Update mount info. Does no DIR scanning.
cbdf862c
WD
1407+ -r, --recurse Scan files in subdirectories too.
1408+ -c, --check Check if the checksums are right (doesn't update).
1409+ -v, --verbose Mention what we're doing. Repeat for more info.
1410+ -h, --help Display this help message.
1411+EOT
1412+}