Use "use warnings" rather than -w on the #! line.
[rsync/rsync-patches.git] / db.diff
CommitLineData
cbdf862c
WD
1Added some DB-access routines to help rsync keep extra filesystem info
2about the files it is dealing with. This adds both the --db=CONFIG_FILE
3option and the "db config" daemon parameter.
4
5For the moment this only adds checksum caching when the --checksum option
bc3fcf1d 6is used. Future improvements may include:
cbdf862c
WD
7
8 - Updating of MD5 checksums when transferring any file, even w/o -c.
bc3fcf1d
WD
9 We should be able to extend this to work for MD4 checksums too if we
10 make the sender force checksum_seed to 0 when using a DB and having
11 the receiving side check to see if it got a 0 checksum_seed. (We
12 probably don't want to compute 2 MD4 checksums for the case where
13 the checksum_seed is non-zero.)
cbdf862c
WD
14
15 - Caching of path info that allows for the finding of files to use for
16 moving/linking/copying/alternate-basis-use.
17
18 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
19
20To use this patch, run these commands for a successful build:
21
22 patch -p1 <patches/remote-option.diff
23 patch -p1 <patches/db.diff
24 ./configure (optional if already run)
25 make
26
27diff --git a/Makefile.in b/Makefile.in
28--- a/Makefile.in
29+++ b/Makefile.in
30@@ -35,7 +35,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
31 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
32 util.o main.o checksum.o match.o syscall.o log.o backup.o
33 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
34- fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
35+ fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
36 OBJS3=progress.o pipe.o
37 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
38 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
39diff --git a/checksum.c b/checksum.c
40--- a/checksum.c
41+++ b/checksum.c
42@@ -23,6 +23,7 @@
43
44 extern int checksum_seed;
45 extern int protocol_version;
46+extern int use_db;
47
48 int csum_length = SHORT_SUM_LENGTH; /* initial value */
49
50@@ -100,10 +101,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
51 }
52 }
53
54-void file_checksum(char *fname, char *sum, OFF_T size)
55+void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
56 {
57 struct map_struct *buf;
58- OFF_T i, len = size;
59+ OFF_T i, len = st_p->st_size;
60 md_context m;
61 int32 remainder;
62 int fd;
63@@ -114,7 +115,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
64 if (fd == -1)
65 return;
66
67- buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
68+ buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
69
70 if (protocol_version >= 30) {
71 md5_begin(&m);
72@@ -148,6 +149,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
73 mdfour_result(&m, (uchar *)sum);
74 }
75
76+ if (use_db)
77+ db_set_checksum(fname, st_p, sum);
78+
79 close(fd);
80 unmap_file(buf);
81 }
82diff --git a/cleanup.c b/cleanup.c
83--- a/cleanup.c
84+++ b/cleanup.c
85@@ -27,6 +27,7 @@ extern int am_daemon;
86 extern int io_error;
87 extern int keep_partial;
88 extern int got_xfer_error;
89+extern int use_db;
90 extern char *partial_dir;
91 extern char *logfile_name;
7f0bf1cb
WD
92
93@@ -124,6 +125,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
cbdf862c
WD
94 /* FALLTHROUGH */
95 #include "case_N.h"
96
97+ if (use_db)
98+ db_disconnect();
99+
100+ /* FALLTHROUGH */
101+#include "case_N.h"
102+
103 if (cleanup_child_pid != -1) {
104 int status;
105 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
106diff --git a/clientserver.c b/clientserver.c
107--- a/clientserver.c
108+++ b/clientserver.c
bc3fcf1d 109@@ -42,13 +42,16 @@ extern int numeric_ids;
cbdf862c
WD
110 extern int filesfrom_fd;
111 extern int remote_protocol;
112 extern int protocol_version;
113+extern int always_checksum;
114 extern int io_timeout;
115 extern int no_detach;
bc3fcf1d 116+extern int use_db;
cbdf862c 117 extern int write_batch;
bc3fcf1d 118 extern int default_af_hint;
cbdf862c
WD
119 extern int logfile_format_has_i;
120 extern int logfile_format_has_o_or_i;
121 extern mode_t orig_umask;
122+extern char *db_config;
123 extern char *bind_address;
124 extern char *sockopts;
125 extern char *config_file;
963ca808 126@@ -564,6 +567,9 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
cbdf862c 127
bc3fcf1d
WD
128 log_init(1);
129
130+ if (*lp_db_config(i))
131+ db_read_config(FLOG, lp_db_config(i));
cbdf862c 132+
bc3fcf1d
WD
133 #ifdef HAVE_PUTENV
134 if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
135 char *modname, *modpath, *hostaddr, *hostname, *username;
abd3adb8 136@@ -767,6 +773,10 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
cbdf862c 137
bc3fcf1d
WD
138 am_server = 1; /* Don't let someone try to be tricky. */
139 quiet = 0;
140+ db_config = NULL;
141+ if (!always_checksum)
142+ use_db = 0;
143+
144 if (lp_ignore_errors(module_id))
145 ignore_errors = 1;
146 if (write_batch < 0)
cbdf862c
WD
147diff --git a/configure.in b/configure.in
148--- a/configure.in
149+++ b/configure.in
bc3fcf1d
WD
150@@ -314,7 +314,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
151 sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
152 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
153 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
154- popt.h popt/popt.h)
155+ popt.h popt/popt.h mysql/mysql.h sqlite3.h)
156 AC_HEADER_MAJOR
157
158 AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[
159@@ -969,6 +969,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
cbdf862c
WD
160 fi
161 fi
162
bc3fcf1d
WD
163+AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
164+if test x$MYSQL_CONFIG = x1; then
165+ AC_MSG_CHECKING(for mysql version >= 4)
166+ mysql_version=`mysql_config --version`
167+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
168+ if test $mysql_major_version -lt 4; then
169+ AC_MSG_RESULT(no.. skipping MySQL)
170+ else
171+ AC_MSG_RESULT(yes)
172+
173+ MYSQL_CFLAGS=`mysql_config --cflags`
174+ MYSQL_LIBS=`mysql_config --libs`
175+
176+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
177+ LIBS="$MYSQL_LIBS $LIBS"
178+
179+ AC_CHECK_LIB(mysqlclient, mysql_init)
180+ fi
181+fi
182+
183+AC_CHECK_LIB(sqlite3, sqlite3_open)
184+AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
cbdf862c
WD
185+
186 case "$CC" in
187 ' checker'*|checker*)
188 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
189diff --git a/db.c b/db.c
190new file mode 100644
191--- /dev/null
192+++ b/db.c
bc3fcf1d 193@@ -0,0 +1,566 @@
cbdf862c
WD
194+/*
195+ * Routines to access extended file info via DB.
196+ *
197+ * Copyright (C) 2008 Wayne Davison
198+ *
199+ * This program is free software; you can redistribute it and/or modify
200+ * it under the terms of the GNU General Public License as published by
201+ * the Free Software Foundation; either version 3 of the License, or
202+ * (at your option) any later version.
203+ *
204+ * This program is distributed in the hope that it will be useful,
205+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
206+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
207+ * GNU General Public License for more details.
208+ *
209+ * You should have received a copy of the GNU General Public License along
210+ * with this program; if not, visit the http://fsf.org website.
211+ */
212+
213+#include "rsync.h"
214+#include "ifuncs.h"
215+
bc3fcf1d 216+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
cbdf862c 217+#define USE_MYSQL
cbdf862c
WD
218+#include <mysql/mysql.h>
219+#include <mysql/errmsg.h>
220+#endif
bc3fcf1d
WD
221+
222+#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
223+#define USE_SQLITE
cbdf862c 224+#include <sqlite3.h>
bc3fcf1d
WD
225+#ifndef HAVE_SQLITE3_OPEN_V2
226+#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
227+ sqlite3_open(dbname, dbhptr)
228+#endif
229+#ifndef HAVE_SQLITE3_PREPARE_V2
230+#define sqlite3_prepare_v2 sqlite3_prepare
231+#endif
cbdf862c
WD
232+#endif
233+
234+extern int protocol_version;
235+extern int checksum_len;
236+
237+#define DB_TYPE_NONE 0
238+#define DB_TYPE_MYSQL 1
239+#define DB_TYPE_SQLITE 2
240+
241+int use_db = DB_TYPE_NONE;
242+
243+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
244+static unsigned int dbport = 0;
245+
246+static union {
247+#ifdef USE_MYSQL
248+ MYSQL *mysql;
249+#endif
250+#ifdef USE_SQLITE
251+ sqlite3 *sqlite;
252+#endif
253+ void *all;
254+} dbh;
255+
256+#define SEL_DEV 0
257+#define SEL_SUM 1
258+#define REP_SUM 2
259+#define MAX_PREP_CNT 3
260+
261+static union {
262+#ifdef USE_MYSQL
263+ MYSQL_STMT *mysql;
264+#endif
265+#ifdef USE_SQLITE
266+ sqlite3_stmt *sqlite;
267+#endif
268+ void *all;
269+} statements[MAX_PREP_CNT];
270+
271+static int md_num;
272+static enum logcode log_code;
273+
bc3fcf1d 274+#ifdef USE_MYSQL
cbdf862c
WD
275+static unsigned int bind_disk_id;
276+static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
bc3fcf1d
WD
277+static char bind_sum[MAX_DIGEST_LEN];
278+#endif
279+static char bind_thishost[256];
cbdf862c
WD
280+static int bind_thishost_len;
281+
282+static unsigned int prior_disk_id = 0;
283+static unsigned long long prior_devno = 0;
284+
285+int db_read_config(enum logcode code, const char *config_file)
286+{
287+ char buf[2048], *cp;
288+ FILE *fp;
289+ int lineno = 0;
290+
291+ log_code = code;
292+
293+ bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
294+
295+ if (!(fp = fopen(config_file, "r"))) {
296+ rsyserr(log_code, errno, "unable to open %s", config_file);
297+ return 0;
298+ }
299+ while (fgets(buf, sizeof buf, fp)) {
300+ lineno++;
301+ if ((cp = strchr(buf, '#')) == NULL
302+ && (cp = strchr(buf, '\r')) == NULL
303+ && (cp = strchr(buf, '\n')) == NULL)
304+ cp = buf + strlen(buf);
305+ while (cp != buf && isSpace(cp-1)) cp--;
306+ *cp = '\0';
307+
308+ if (!*buf)
309+ continue;
310+
311+ if (!(cp = strchr(buf, ':')))
312+ goto invalid_line;
313+ *cp++ = '\0';
314+
315+ while (isSpace(cp)) cp++;
316+ if (strcasecmp(buf, "dbhost") == 0)
317+ dbhost = strdup(cp);
318+ else if (strcasecmp(buf, "dbuser") == 0)
319+ dbuser = strdup(cp);
320+ else if (strcasecmp(buf, "dbpass") == 0)
321+ dbpass = strdup(cp);
322+ else if (strcasecmp(buf, "dbname") == 0)
323+ dbname = strdup(cp);
324+ else if (strcasecmp(buf, "dbport") == 0)
325+ dbport = atoi(cp);
326+ else if (strcasecmp(buf, "thishost") == 0)
327+ bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
328+ else if (strcasecmp(buf, "dbtype") == 0) {
329+#ifdef USE_MYSQL
330+ if (strcasecmp(cp, "mysql") == 0) {
331+ use_db = DB_TYPE_MYSQL;
332+ continue;
333+ }
334+#endif
335+#ifdef USE_SQLITE
336+ if (strcasecmp(cp, "sqlite") == 0) {
337+ use_db = DB_TYPE_SQLITE;
338+ continue;
339+ }
340+#endif
341+ rprintf(log_code,
342+ "Unsupported dbtype on line #%d in %s.\n",
343+ lineno, config_file);
344+ use_db = DB_TYPE_NONE;
345+ return 0;
346+ } else {
347+ invalid_line:
348+ rprintf(log_code, "Invalid line #%d in %s\n",
349+ lineno, config_file);
350+ use_db = DB_TYPE_NONE;
351+ return 0;
352+ }
353+ }
354+ fclose(fp);
355+
356+ if (bind_thishost_len >= (int)sizeof bind_thishost)
357+ bind_thishost_len = sizeof bind_thishost - 1;
358+
359+ if (!use_db || !dbname) {
360+ rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
361+ use_db = DB_TYPE_NONE;
362+ return 0;
363+ }
364+
365+ md_num = protocol_version >= 30 ? 5 : 4;
366+
367+ return 1;
368+}
369+
370+#ifdef USE_MYSQL
371+static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
372+{
373+ va_list ap;
374+ char *query;
375+ int qlen, param_cnt;
376+ MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
377+
378+ if (stmt == NULL)
379+ out_of_memory("prepare_mysql");
380+
381+ va_start(ap, fmt);
382+ qlen = vasprintf(&query, fmt, ap);
383+ va_end(ap);
384+ if (qlen < 0)
385+ out_of_memory("prepare_mysql");
386+
387+ if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
388+ rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
389+ return NULL;
390+ }
391+ free(query);
392+
393+ if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
394+ rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
395+ param_cnt, bind_cnt);
396+ return NULL;
397+ }
398+ if (bind_cnt)
399+ mysql_stmt_bind_param(stmt, binds);
400+
401+ return stmt;
402+}
403+#endif
404+
405+#ifdef USE_MYSQL
406+static int db_connect_mysql(void)
407+{
408+ MYSQL_BIND binds[10];
409+
410+ if (!(dbh.mysql = mysql_init(NULL)))
411+ out_of_memory("db_read_config");
412+
413+ if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
414+ return 0;
415+
416+ memset(binds, 0, sizeof binds);
417+ binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
418+ binds[0].buffer = &bind_devno;
419+ binds[1].buffer_type = MYSQL_TYPE_STRING;
420+ binds[1].buffer = &bind_thishost;
421+ binds[1].buffer_length = bind_thishost_len;
422+ statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
423+ "SELECT disk_id"
424+ " FROM disk"
425+ " WHERE devno = ? AND host = ? AND mounted = 1");
426+ if (!statements[SEL_DEV].mysql)
427+ return 0;
428+
429+ memset(binds, 0, sizeof binds);
430+ binds[0].buffer_type = MYSQL_TYPE_LONG;
431+ binds[0].buffer = &bind_disk_id;
432+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
433+ binds[1].buffer = &bind_ino;
434+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
435+ binds[2].buffer = &bind_size;
436+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
437+ binds[3].buffer = &bind_mtime;
438+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
439+ binds[4].buffer = &bind_ctime;
440+ statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
441+ "SELECT checksum"
442+ " FROM inode_map"
443+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
444+ " AND size = ? AND mtime = ? AND ctime = ?",
445+ md_num);
446+ if (!statements[SEL_SUM].mysql)
447+ return 0;
448+
449+ memset(binds, 0, sizeof binds);
450+ binds[0].buffer_type = MYSQL_TYPE_LONG;
451+ binds[0].buffer = &bind_disk_id;
452+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
453+ binds[1].buffer = &bind_ino;
454+ binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
455+ binds[2].buffer = binds[6].buffer = &bind_size;
456+ binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
457+ binds[3].buffer = binds[7].buffer = &bind_mtime;
458+ binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
459+ binds[4].buffer = binds[8].buffer = &bind_ctime;
460+ binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
461+ binds[5].buffer = binds[9].buffer = &bind_sum;
462+ binds[5].buffer_length = binds[9].buffer_length = checksum_len;
463+ statements[REP_SUM].mysql = prepare_mysql(binds, 10,
464+ "INSERT INTO inode_map"
465+ " SET disk_id = ?, ino = ?, sum_type = %d,"
466+ " size = ?, mtime = ?, ctime = ?, checksum = ?"
467+ " ON DUPLICATE KEY"
468+ " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
469+ md_num, md_num);
470+ if (!statements[REP_SUM].mysql)
471+ return 0;
472+
473+ return 1;
474+}
475+#endif
476+
477+#ifdef USE_SQLITE
478+static int db_connect_sqlite(void)
479+{
480+ char *sql;
481+
cbdf862c
WD
482+ if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
483+ return 0;
cbdf862c
WD
484+
485+ sql = "SELECT disk_id"
486+ " FROM disk"
487+ " WHERE devno = ? AND host = ? AND mounted = 1";
488+ if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
489+ return 0;
490+
491+ if (asprintf(&sql,
492+ "SELECT checksum"
493+ " FROM inode_map"
494+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
495+ " AND size = ? AND mtime = ? AND ctime = ?",
496+ md_num) < 0
497+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
498+ return 0;
499+ free(sql);
500+
501+ if (asprintf(&sql,
502+ "INSERT OR REPLACE INTO inode_map"
503+ " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
504+ " VALUES(?, ?, %d, ?, ?, ?, ?)",
505+ md_num) < 0
506+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
507+ return 0;
508+ free(sql);
509+
510+ return 1;
511+}
512+#endif
513+
514+int db_connect(void)
515+{
516+ switch (use_db) {
517+#ifdef USE_MYSQL
518+ case DB_TYPE_MYSQL:
519+ if (db_connect_mysql())
520+ return 1;
521+ break;
522+#endif
523+#ifdef USE_SQLITE
524+ case DB_TYPE_SQLITE:
525+ if (db_connect_sqlite())
526+ return 1;
527+ break;
528+#endif
529+ }
530+
531+ rprintf(log_code, "Unable to connect to DB\n");
532+ db_disconnect();
533+ use_db = DB_TYPE_NONE;
534+
535+ return 0;
536+}
537+
538+void db_disconnect(void)
539+{
540+ int ndx;
541+
542+ if (!dbh.all)
543+ return;
544+
545+ for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
546+ if (statements[ndx].all) {
547+ switch (use_db) {
548+#ifdef USE_MYSQL
549+ case DB_TYPE_MYSQL:
550+ mysql_stmt_close(statements[ndx].mysql);
551+ break;
552+#endif
553+#ifdef USE_SQLITE
554+ case DB_TYPE_SQLITE:
555+ sqlite3_finalize(statements[ndx].sqlite);
556+ break;
557+#endif
558+ }
559+ statements[ndx].all = NULL;
560+ }
561+ }
562+
563+ switch (use_db) {
564+#ifdef USE_MYSQL
565+ case DB_TYPE_MYSQL:
566+ mysql_close(dbh.mysql);
567+ break;
568+#endif
569+#ifdef USE_SQLITE
570+ case DB_TYPE_SQLITE:
571+ sqlite3_close(dbh.sqlite);
572+ break;
573+#endif
574+ }
575+
576+ dbh.all = NULL;
577+}
578+
bc3fcf1d 579+#ifdef USE_MYSQL
cbdf862c
WD
580+static MYSQL_STMT *exec_mysql(int ndx)
581+{
582+ MYSQL_STMT *stmt = statements[ndx].mysql;
583+ int rc;
584+
585+ if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
586+ db_disconnect();
587+ if (db_connect()) {
588+ stmt = statements[ndx].mysql;
589+ rc = mysql_stmt_execute(stmt);
590+ }
591+ }
592+ if (rc != 0) {
593+ rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
594+ return NULL;
595+ }
596+
597+ return stmt;
598+}
bc3fcf1d 599+#endif
cbdf862c 600+
bc3fcf1d 601+#ifdef USE_MYSQL
cbdf862c
WD
602+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
603+{
604+ unsigned long length[32];
605+ my_bool is_null[32], error[32];
606+ MYSQL_STMT *stmt;
607+ int i, rc;
608+
609+ if (bind_cnt > 32)
610+ exit_cleanup(RERR_UNSUPPORTED);
611+
612+ if ((stmt = exec_mysql(ndx)) == NULL)
613+ return 0;
614+
615+ for (i = 0; i < bind_cnt; i++) {
616+ binds[i].is_null = &is_null[i];
617+ binds[i].length = &length[i];
618+ binds[i].error = &error[i];
619+ }
620+ mysql_stmt_bind_result(stmt, binds);
621+
622+ if ((rc = mysql_stmt_fetch(stmt)) != 0) {
623+ if (rc != MYSQL_NO_DATA) {
624+ rprintf(log_code, "SELECT fetch failed: %s\n",
625+ mysql_stmt_error(stmt));
626+ }
627+ mysql_stmt_free_result(stmt);
628+ return 0;
629+ }
630+
631+ mysql_stmt_free_result(stmt);
632+
633+ return is_null[0] ? 0 : 1;
634+}
bc3fcf1d 635+#endif
cbdf862c
WD
636+
637+static void get_disk_id(unsigned long long devno)
638+{
639+ switch (use_db) {
640+#ifdef USE_MYSQL
641+ case DB_TYPE_MYSQL: {
642+ MYSQL_BIND binds[1];
643+
644+ bind_devno = devno; /* The one variable SEL_DEV input value. */
645+
646+ /* Bind where to put the output. */
647+ binds[0].buffer_type = MYSQL_TYPE_LONG;
648+ binds[0].buffer = &prior_disk_id;
649+ if (!fetch_mysql(binds, 1, SEL_DEV))
650+ prior_disk_id = 0;
651+ break;
652+ }
653+#endif
654+#ifdef USE_SQLITE
655+ case DB_TYPE_SQLITE: {
656+ sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
657+ sqlite3_bind_int64(stmt, 1, devno);
658+ sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
659+ if (sqlite3_step(stmt) == SQLITE_ROW)
660+ prior_disk_id = sqlite3_column_int(stmt, 0);
661+ else
662+ prior_disk_id = 0;
663+ sqlite3_reset(stmt);
664+ break;
665+ }
666+#endif
667+ }
668+
669+ prior_devno = devno;
670+}
671+
672+int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
673+{
674+ if (prior_devno != st_p->st_dev)
675+ get_disk_id(st_p->st_dev);
676+ if (prior_disk_id == 0)
677+ return 0;
678+
679+ switch (use_db) {
680+#ifdef USE_MYSQL
681+ case DB_TYPE_MYSQL: {
682+ MYSQL_BIND binds[1];
683+
684+ bind_disk_id = prior_disk_id;
685+ bind_ino = st_p->st_ino;
686+ bind_size = st_p->st_size;
687+ bind_mtime = st_p->st_mtime;
688+ bind_ctime = st_p->st_ctime;
689+
690+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
691+ binds[0].buffer = sum;
692+ binds[0].buffer_length = checksum_len;
693+ return fetch_mysql(binds, 1, SEL_SUM);
694+ }
695+#endif
696+#ifdef USE_SQLITE
697+ case DB_TYPE_SQLITE: {
698+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
699+ sqlite3_bind_int(stmt, 1, prior_disk_id);
700+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
701+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
702+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
703+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
704+ if (sqlite3_step(stmt) == SQLITE_ROW) {
705+ int len = sqlite3_column_bytes(stmt, 0);
706+ if (len > MAX_DIGEST_LEN)
707+ len = MAX_DIGEST_LEN;
708+ memcpy(sum, sqlite3_column_blob(stmt, 0), len);
709+ sqlite3_reset(stmt);
710+ return 1;
711+ }
712+ sqlite3_reset(stmt);
713+ return 0;
714+ }
715+#endif
716+ }
717+
718+ return 0;
719+}
720+
721+int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
722+{
723+ if (prior_devno != st_p->st_dev)
724+ get_disk_id(st_p->st_dev);
725+ if (prior_disk_id == 0)
726+ return 0;
727+
728+ switch (use_db) {
729+#ifdef USE_MYSQL
730+ case DB_TYPE_MYSQL: {
731+ bind_disk_id = prior_disk_id;
732+ bind_ino = st_p->st_ino;
733+ bind_size = st_p->st_size;
734+ bind_mtime = st_p->st_mtime;
735+ bind_ctime = st_p->st_ctime;
736+ memcpy(bind_sum, sum, checksum_len);
737+
738+ return exec_mysql(REP_SUM) != NULL;
739+ }
740+#endif
741+#ifdef USE_SQLITE
742+ case DB_TYPE_SQLITE: {
743+ int rc;
744+ sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
745+ sqlite3_bind_int(stmt, 1, prior_disk_id);
746+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
747+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
748+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
749+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
750+ sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
751+ rc = sqlite3_step(stmt);
752+ sqlite3_reset(stmt);
753+ return rc == SQLITE_DONE;
754+ }
755+#endif
756+ }
757+
758+ return 0;
759+}
760diff --git a/flist.c b/flist.c
761--- a/flist.c
762+++ b/flist.c
763@@ -54,6 +54,7 @@ extern int preserve_devices;
764 extern int preserve_specials;
765 extern int uid_ndx;
766 extern int gid_ndx;
767+extern int use_db;
768 extern int eol_nulls;
769 extern int relative_paths;
770 extern int implied_dirs;
ae306a29 771@@ -1286,14 +1287,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
cbdf862c
WD
772 memcpy(bp + basename_len, linkname, linkname_len);
773 #endif
774
775- if (always_checksum && am_sender && S_ISREG(st.st_mode))
776- file_checksum(thisname, tmp_sum, st.st_size);
777-
778 if (am_sender)
779 F_PATHNAME(file) = pathname;
780 else if (!pool)
781 F_DEPTH(file) = extra_len / EXTRA_LEN;
782
783+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
784+ if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
785+ file_checksum(thisname, &st, tmp_sum);
786+ }
787+
abd3adb8
WD
788 if (basename_len == 0+1) {
789 if (!pool)
790 unmake_file(file);
bf1bd9d4 791@@ -1936,6 +1939,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
cbdf862c
WD
792 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
793 int implied_dot_dir = 0;
794
795+ if (use_db)
796+ db_connect();
797+
798 rprintf(FLOG, "building file list\n");
799 if (show_filelist_p())
800 start_filelist_progress("building file list");
801diff --git a/generator.c b/generator.c
802--- a/generator.c
803+++ b/generator.c
804@@ -58,6 +58,7 @@ extern int update_only;
805 extern int ignore_existing;
806 extern int ignore_non_existing;
807 extern int inplace;
808+extern int use_db;
809 extern int append_mode;
810 extern int make_backups;
811 extern int csum_length;
abd3adb8 812@@ -724,7 +725,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
cbdf862c
WD
813 of the file time to determine whether to sync */
814 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
815 char sum[MAX_DIGEST_LEN];
816- file_checksum(fn, sum, st->st_size);
817+ if (!use_db || !db_get_checksum(fn, st, sum))
818+ file_checksum(fn, st, sum);
819 return memcmp(sum, F_SUM(file), checksum_len) == 0;
820 }
821
abd3adb8 822@@ -2206,6 +2208,9 @@ void generate_files(int f_out, const char *local_name)
cbdf862c
WD
823 : "enabled");
824 }
825
826+ if (use_db && always_checksum)
827+ db_connect();
828+
829 /* Since we often fill up the outgoing socket and then just sit around
830 * waiting for the other 2 processes to do their thing, we don't want
831 * to exit on a timeout. If the data stops flowing, the receiver will
832diff --git a/loadparm.c b/loadparm.c
833--- a/loadparm.c
834+++ b/loadparm.c
835@@ -126,6 +126,7 @@ typedef struct
836 char *auth_users;
837 char *charset;
838 char *comment;
839+ char *db_config;
840 char *dont_compress;
841 char *exclude;
842 char *exclude_from;
843@@ -177,6 +178,7 @@ static service sDefault =
844 /* auth_users; */ NULL,
845 /* charset; */ NULL,
846 /* comment; */ NULL,
847+ /* db_config; */ NULL,
848 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
849 /* exclude; */ NULL,
850 /* exclude_from; */ NULL,
851@@ -307,6 +309,7 @@ static struct parm_struct parm_table[] =
852 {"auth users", P_STRING, P_LOCAL, &sDefault.auth_users, NULL,0},
853 {"charset", P_STRING, P_LOCAL, &sDefault.charset, NULL,0},
854 {"comment", P_STRING, P_LOCAL, &sDefault.comment, NULL,0},
855+ {"db config", P_STRING, P_LOCAL, &sDefault.db_config, NULL,0},
856 {"dont compress", P_STRING, P_LOCAL, &sDefault.dont_compress, NULL,0},
857 {"exclude from", P_STRING, P_LOCAL, &sDefault.exclude_from, NULL,0},
858 {"exclude", P_STRING, P_LOCAL, &sDefault.exclude, NULL,0},
859@@ -400,6 +403,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Globals.rsync_port)
860 FN_LOCAL_STRING(lp_auth_users, auth_users)
861 FN_LOCAL_STRING(lp_charset, charset)
862 FN_LOCAL_STRING(lp_comment, comment)
863+FN_LOCAL_STRING(lp_db_config, db_config)
864 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
865 FN_LOCAL_STRING(lp_exclude, exclude)
866 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
867diff --git a/main.c b/main.c
868--- a/main.c
869+++ b/main.c
870@@ -49,6 +49,7 @@ extern int copy_unsafe_links;
871 extern int keep_dirlinks;
872 extern int preserve_hard_links;
873 extern int protocol_version;
874+extern int always_checksum;
875 extern int file_total;
876 extern int recurse;
877 extern int xfer_dirs;
4c107044 878@@ -73,6 +74,7 @@ extern char *partial_dir;
cbdf862c
WD
879 extern char *dest_option;
880 extern char *basis_dir[];
881 extern char *rsync_path;
882+extern char *db_config;
883 extern char *shell_cmd;
884 extern char *batch_name;
885 extern char *password_file;
4c107044 886@@ -1491,6 +1493,9 @@ int main(int argc,char *argv[])
cbdf862c
WD
887 exit_cleanup(RERR_SYNTAX);
888 }
889
bc3fcf1d 890+ if (db_config && always_checksum)
cbdf862c
WD
891+ db_read_config(FERROR, db_config);
892+
893 if (am_server) {
894 set_nonblocking(STDIN_FILENO);
895 set_nonblocking(STDOUT_FILENO);
896diff --git a/options.c b/options.c
897--- a/options.c
898+++ b/options.c
899@@ -92,6 +92,7 @@ int use_qsort = 0;
900 char *files_from = NULL;
901 int filesfrom_fd = -1;
902 char *filesfrom_host = NULL;
903+char *db_config = NULL;
904 int eol_nulls = 0;
905 int protect_args = 0;
906 int human_readable = 0;
bc3fcf1d
WD
907@@ -229,6 +230,7 @@ static void print_rsync_version(enum logcode f)
908 char const *links = "no ";
909 char const *iconv = "no ";
910 char const *ipv6 = "no ";
911+ char const *db = "no ";
912 STRUCT_STAT *dumstat;
913
914 #if SUBPROTOCOL_VERSION != 0
abd3adb8 915@@ -262,6 +264,11 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
916 #if defined HAVE_LUTIMES && defined HAVE_UTIMES
917 symtimes = "";
918 #endif
919+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
920+ db = "";
921+#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
922+ db = "";
923+#endif
924
925 rprintf(f, "%s version %s protocol version %d%s\n",
926 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
abd3adb8 927@@ -275,8 +282,8 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
928 (int)(sizeof (int64) * 8));
929 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
930 got_socketpair, hardlinks, links, ipv6, have_inplace);
931- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
932- have_inplace, acls, xattrs, iconv, symtimes);
933+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n",
934+ have_inplace, acls, xattrs, iconv, symtimes, db);
935
936 #ifdef MAINTAINER_MODE
937 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
abd3adb8 938@@ -322,6 +329,7 @@ void usage(enum logcode F)
cbdf862c
WD
939 rprintf(F," -q, --quiet suppress non-error messages\n");
940 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
941 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
bc3fcf1d 942+ rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
cbdf862c
WD
943 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
944 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
945 rprintf(F," -r, --recursive recurse into directories\n");
abd3adb8 946@@ -580,6 +588,7 @@ static struct poptOption long_options[] = {
cbdf862c
WD
947 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
948 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
949 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
950+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
951 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
952 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
953 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
954diff --git a/pipe.c b/pipe.c
955--- a/pipe.c
956+++ b/pipe.c
bc3fcf1d 957@@ -26,6 +26,9 @@ extern int am_sender;
cbdf862c
WD
958 extern int am_server;
959 extern int blocking_io;
960 extern int filesfrom_fd;
961+extern int always_checksum;
cbdf862c
WD
962+extern int use_db;
963+extern char *db_config;
964 extern mode_t orig_umask;
965 extern char *logfile_name;
966 extern int remote_option_cnt;
bc3fcf1d 967@@ -141,6 +144,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
968 logfile_close();
969 }
970
971+ use_db = 0;
972+ db_config = NULL;
973+
974 if (remote_option_cnt) {
975 int rc = remote_option_cnt + 1;
976 const char **rv = remote_options;
bc3fcf1d 977@@ -148,6 +154,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
978 option_error();
979 exit_cleanup(RERR_SYNTAX);
980 }
bc3fcf1d 981+ if (db_config && always_checksum)
cbdf862c
WD
982+ db_read_config(FERROR, db_config);
983 }
984
985 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
bc3fcf1d
WD
986diff --git a/rsync.yo b/rsync.yo
987--- a/rsync.yo
988+++ b/rsync.yo
abd3adb8 989@@ -321,6 +321,7 @@ to the detailed description below for a complete description. verb(
bc3fcf1d
WD
990 -q, --quiet suppress non-error messages
991 --no-motd suppress daemon-mode MOTD (see caveat)
992 -c, --checksum skip based on checksum, not mod-time & size
993+ --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
994 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
995 --no-OPTION turn off an implied OPTION (e.g. --no-D)
996 -r, --recursive recurse into directories
abd3adb8 997@@ -537,6 +538,47 @@ checksum that is generated as the file is transferred, but that
bc3fcf1d
WD
998 automatic after-the-transfer verification has nothing to do with this
999 option's before-the-transfer "Does this file need to be updated?" check.
cbdf862c 1000
bc3fcf1d
WD
1001+dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
1002+that holds connection details for a database of checksum information.
1003+When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
1004+use cached checksum information from the DB, and will update it if it is
1005+missing.
1006+
1007+The currently supported DB choices are MySQL and SQLite. For example, a
1008+MySQL configuration might look like this:
1009+
1010+verb( dbtype: mysql
1011+ dbhost: 127.0.0.1
1012+ dbname: rsyncdb
1013+ dbuser: rsyncuser
1014+ dbpass: somepass
1015+ port: 3306
1016+ thishost: hostname )
1017+
1018+And a SQLite configuration might look like this:
1019+
1020+verb( dbtype: SQLite
1021+ dbname: /var/cache/rsync/sum.db )
1022+
1023+This option only affects one side of a transfer. See the
1024+bf(--remote-option) option for a way to specify the option for both
1025+sides of the transfer (with each side reading the config file from
1026+their local filesystem). For example:
1027+
1028+verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
1029+
1030+See the perl script "rsyncdb" in the support directory of the source code
1031+(which may also be installed in /usr/bin) for a way to create the tables,
1032+populate the mounted-disk information, check files against their checksums,
1033+and update both the MD4 and MD5 checksums for files at the same time (since
1034+an rsync copy will only update one or the other).
1035+
1036+You can use a single MySQL DB for all your hosts if you give each one
1037+their own "thishost" name and setup their device-mapping data. Or feel
1038+free to use separate databases, separate servers, etc. See the rsync
1039+daemon's "db config" parameter for how to configure a daemon to use a DB
1040+(since a client cannot control this parameter on a daemon).
1041+
1042 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
1043 way of saying you want recursion and want to preserve almost
1044 everything (with -H being a notable omission).
1045diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
1046--- a/rsyncd.conf.yo
1047+++ b/rsyncd.conf.yo
1048@@ -270,6 +270,18 @@ is daemon. This setting has no effect if the "log file" setting is a
1049 non-empty string (either set in the per-modules settings, or inherited
1050 from the global settings).
cbdf862c 1051
bc3fcf1d
WD
1052+dit(bf(db config)) This parameter specifies a config file to read that
1053+holds connection details for a database of checksum information.
1054+
1055+The config file will be read-in prior to any chroot restrictions, but
1056+the connection occurs from inside the chroot. This means that you
1057+should use a socket connection (e.g. 127.0.0.1 rather than localhost)
1058+for a MySQL config from inside a chroot. For SQLite, the DB file must
1059+be placed inside the chroot (though it can be placed outside the
1060+transfer dir if you configured an inside-chroot path).
1061+
1062+See the bf(--db=CONFIG_FILE) option for full details.
1063+
e66d6d51 1064 dit(bf(max verbosity)) This parameter allows you to control
bc3fcf1d
WD
1065 the maximum amount of verbose information that you'll allow the daemon to
1066 generate (since the information goes into the log file). The default is 1,
1067diff --git a/support/rsyncdb b/support/rsyncdb
cbdf862c
WD
1068new file mode 100755
1069--- /dev/null
bc3fcf1d 1070+++ b/support/rsyncdb
0ef5abcb
WD
1071@@ -0,0 +1,332 @@
1072+#!/usr/bin/perl
cbdf862c 1073+use strict;
0ef5abcb 1074+use warnings;
cbdf862c
WD
1075+
1076+use DBI;
1077+use Getopt::Long;
1078+use Cwd qw(abs_path cwd);
1079+use Digest::MD4;
1080+use Digest::MD5;
1081+
1082+my $MOUNT_FILE = '/etc/mtab';
1083+
1084+&Getopt::Long::Configure('bundling');
1085+&usage if !&GetOptions(
1086+ 'db=s' => \( my $db_config ),
bc3fcf1d 1087+ 'init' => \( my $init_db ),
cbdf862c
WD
1088+ 'mounts|m' => \( my $update_mounts ),
1089+ 'recurse|r' => \( my $recurse_opt ),
1090+ 'check|c' => \( my $check_opt ),
1091+ 'verbose|v+' => \( my $verbosity = 0 ),
1092+ 'help|h' => \( my $help_opt ),
1093+);
1094+&usage if $help_opt || !defined $db_config;
1095+
1096+my %config;
1097+open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
1098+while (<IN>) {
1099+ s/[#\r\n].*//s;
1100+ next if /^$/;
1101+ my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
1102+ $config{$key} = $val;
1103+}
1104+close IN;
1105+
1106+die "You must define at least dbtype and dbname in $db_config\n"
1107+ unless defined $config{'dbtype'} && defined $config{'dbname'};
1108+
bc3fcf1d
WD
1109+my $sqlite = $config{'dbtype'} =~ /^sqlite$/i;
1110+
cbdf862c
WD
1111+my $thishost = $config{'thishost'} || 'localhost';
1112+
bc3fcf1d
WD
1113+my $connect = 'DBI:' . $config{'dbtype'} . ':';
1114+$connect .= 'dbname=' . $config{'dbname'} if $sqlite;
1115+$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db;
cbdf862c
WD
1116+$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
1117+$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
1118+
1119+my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
1120+ or die "DB connection failed\n";
1121+
1122+END {
1123+ $dbh->disconnect if defined $dbh;
1124+}
1125+
bc3fcf1d
WD
1126+if ($init_db) {
1127+ my $unsigned = $sqlite ? '' : 'unsigned';
1128+ my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT';
1129+ my $dbname = $config{'dbname'};
1130+
1131+ if (!$sqlite) {
1132+ $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`");
1133+ $dbh->do("USE `$dbname`");
1134+ }
1135+
1136+ print "Dropping old tables (if they exist) ...\n" if $verbosity;
1137+ $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr;
1138+ $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr;
1139+
1140+ print "Creating empty tables ...\n" if $verbosity;
1141+ $dbh->do("
1142+ CREATE TABLE disk (
1143+ disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment,
1144+ devno bigint $unsigned NOT NULL,
1145+ host varchar(256) NOT NULL default 'localhost',
1146+ mounted tinyint NOT NULL default '1',
1147+ comment varchar(256) default NULL
1148+ )") or die $dbh->errstr;
1149+
1150+ $dbh->do("
1151+ CREATE TABLE inode_map (
1152+ disk_id integer $unsigned NOT NULL,
1153+ ino bigint $unsigned NOT NULL,
1154+ size bigint $unsigned NOT NULL,
1155+ mtime bigint NOT NULL,
1156+ ctime bigint NOT NULL,
1157+ sum_type tinyint NOT NULL default '0',
1158+ checksum binary(16) NOT NULL,
1159+ PRIMARY KEY (disk_id,ino,sum_type)
1160+ )") or die $dbh->errstr;
1161+
1162+ exit unless $update_mounts;
1163+}
1164+
cbdf862c
WD
1165+my $sel_disk_H = $dbh->prepare("
1166+ SELECT disk_id, devno, mounted, comment
1167+ FROM disk
1168+ WHERE host = ?
1169+ ") or die $dbh->errstr;
1170+
1171+my $ins_disk_H = $dbh->prepare("
1172+ INSERT INTO disk
1173+ (devno, host, mounted, comment)
1174+ VALUES(?, ?, ?, ?)
1175+ ") or die $dbh->errstr;
1176+
1177+my $up_disk_H = $dbh->prepare("
1178+ UPDATE disk
1179+ SET mounted = ?
1180+ WHERE disk_id = ?
1181+ ") or die $dbh->errstr;
1182+
bc3fcf1d 1183+my $row_id = $sqlite ? 'ROWID' : 'ID';
cbdf862c
WD
1184+my $sel_lastid_H = $dbh->prepare("
1185+ SELECT LAST_INSERT_$row_id()
1186+ ") or die $dbh->errstr;
1187+
1188+my $sel_sum_H = $dbh->prepare("
1189+ SELECT sum_type, checksum
1190+ FROM inode_map
1191+ WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
1192+ ") or die $dbh->errstr;
1193+
1194+my $rep_sum_H = $dbh->prepare("
1195+ REPLACE INTO inode_map
1196+ (disk_id, ino, size, mtime, ctime, sum_type, checksum)
1197+ VALUES(?, ?, ?, ?, ?, ?, ?)
1198+ ") or die $dbh->errstr;
1199+
1200+my %mounts;
1201+if ($update_mounts) {
1202+ open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
1203+ while (<IN>) {
1204+ my($devname, $mnt) = (split)[0,1];
1205+ next unless $devname =~ m#^/dev#;
1206+ my($devno) = (stat($mnt))[0];
1207+ if (!defined $devno) {
1208+ warn "Unable to stat $mnt: $!\n";
1209+ next;
1210+ }
1211+ $mounts{$devno} = "$devname on $mnt";
1212+ }
1213+ close IN;
1214+}
1215+
1216+my %disk_id;
1217+$sel_disk_H->execute($thishost);
1218+while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
1219+ if ($update_mounts) {
1220+ if (defined $mounts{$devno}) {
1221+ if ($comment ne $mounts{$devno}) {
1222+ if ($mounted) {
bc3fcf1d 1223+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1224+ $up_disk_H->execute(0, $disk_id);
1225+ }
1226+ next;
1227+ }
1228+ if (!$mounted) {
bc3fcf1d 1229+ print "Mounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1230+ $up_disk_H->execute(1, $disk_id);
1231+ }
1232+ } else {
1233+ if ($mounted) {
bc3fcf1d 1234+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1235+ $up_disk_H->execute(0, $disk_id);
1236+ }
1237+ next;
1238+ }
1239+ } else {
1240+ next unless $mounted;
1241+ }
1242+ $disk_id{$devno} = $disk_id;
1243+}
1244+$sel_disk_H->finish;
1245+
1246+if ($update_mounts) {
1247+ while (my($devno, $comment) = each %mounts) {
1248+ next if $disk_id{$devno};
bc3fcf1d 1249+ print "Adding $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1250+ $ins_disk_H->execute($devno, $thishost, 1, $comment);
1251+ $sel_lastid_H->execute;
1252+ ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
1253+ $sel_lastid_H->finish;
1254+ }
bc3fcf1d 1255+ exit;
cbdf862c
WD
1256+}
1257+
1258+my $start_dir = cwd();
1259+
1260+my @dirs = @ARGV;
1261+@dirs = '.' unless @dirs;
1262+foreach (@dirs) {
1263+ $_ = abs_path($_);
1264+}
1265+
1266+$| = 1;
1267+
1268+my $exit_code = 0;
1269+
1270+my $md4 = Digest::MD4->new;
1271+my $md5 = Digest::MD5->new;
1272+
1273+while (@dirs) {
1274+ my $dir = shift @dirs;
1275+
1276+ if (!chdir($dir)) {
1277+ warn "Unable to chdir to $dir: $!\n";
1278+ next;
1279+ }
1280+ if (!opendir(DP, '.')) {
1281+ warn "Unable to opendir $dir: $!\n";
1282+ next;
1283+ }
1284+
1285+ my $reldir = $dir;
1286+ $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
1287+ print "$reldir ... \n" if $verbosity;
1288+
1289+ my @subdirs;
1290+ while (defined(my $fn = readdir(DP))) {
1291+ next if $fn =~ /^\.\.?$/ || -l $fn;
1292+ if (-d _) {
1293+ push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
1294+ next;
1295+ }
1296+ next unless -f _;
1297+
1298+ my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
1299+ my $disk_id = $disk_id{$dev} or next;
1300+ $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
1301+ my($sum4, $dbsum4, $sum5, $dbsum5);
1302+ my $dbsumcnt = 0;
1303+ while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
1304+ if ($sum_type == 4) {
1305+ $dbsum4 = $checksum;
1306+ $dbsumcnt++;
1307+ } elsif ($sum_type == 5) {
1308+ $dbsum5 = $checksum;
1309+ $dbsumcnt++;
1310+ }
1311+ }
1312+ $sel_sum_H->finish;
1313+
1314+ next if !$check_opt && $dbsumcnt == 2;
1315+
1316+ if (!$check_opt || $dbsumcnt || $verbosity > 2) {
1317+ if (!open(IN, $fn)) {
1318+ print STDERR "Unable to read $fn: $!\n";
1319+ next;
1320+ }
1321+
1322+ while (1) {
1323+ while (sysread(IN, $_, 64*1024)) {
1324+ $md4->add($_);
1325+ $md5->add($_);
1326+ }
1327+ $sum4 = $md4->digest;
1328+ $sum5 = $md5->digest;
1329+ print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
1330+ print " $fn" if $verbosity > 1;
1331+ my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
1332+ last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
1333+ $ino = $ino2;
1334+ $size = $size2;
1335+ $mtime = $mtime2;
1336+ $ctime = $ctime2;
1337+ sysseek(IN, 0, 0);
1338+ print " REREADING\n" if $verbosity > 1;
1339+ }
1340+
1341+ close IN;
1342+ } elsif ($verbosity > 1) {
1343+ print "_$fn";
1344+ }
1345+
1346+ if ($check_opt) {
1347+ my $dif;
1348+ if ($dbsumcnt == 0) {
1349+ $dif = ' --MISSING--';
1350+ } else {
1351+ $dif = '';
1352+ if (!defined $dbsum4) {
1353+ $dif .= ' -NO-MD4-';
1354+ } elsif ($sum4 ne $dbsum4) {
1355+ $dif .= ' -MD4-CHANGED-';
1356+ }
1357+ if (!defined $dbsum5) {
1358+ $dif .= ' ---NO-MD5---';
1359+ } elsif ($sum5 ne $dbsum5) {
1360+ $dif .= ' -MD5-CHANGED-';
1361+ }
1362+ if ($dif eq '') {
1363+ print " ====OK====\n" if $verbosity > 1;
1364+ next;
1365+ }
1366+ $dif =~ s/MD4-CHANGED MD5-//;
1367+ }
1368+ if ($verbosity < 2) {
1369+ print $verbosity ? ' ' : "$reldir/";
1370+ print $fn;
1371+ }
1372+ print $dif, "\n";
1373+ $exit_code = 1;
1374+ } else {
1375+ print "\n" if $verbosity > 1;
1376+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
1377+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
1378+ }
1379+ }
1380+
1381+ closedir DP;
1382+
1383+ unshift(@dirs, sort @subdirs) if $recurse_opt;
1384+}
1385+
1386+exit $exit_code;
1387+
1388+sub usage
1389+{
1390+ die <<EOT;
1391+Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
1392+
1393+Options:
1394+ --db=FILE Specify the config FILE to read for the DB info.
bc3fcf1d
WD
1395+ --init Create (recreate) needed tables (making them empty).
1396+ No DIR scanning, but can be combined with --mounts.
1397+ -m, --mounts Update mount info. Does no DIR scanning.
cbdf862c
WD
1398+ -r, --recurse Scan files in subdirectories too.
1399+ -c, --check Check if the checksums are right (doesn't update).
1400+ -v, --verbose Mention what we're doing. Repeat for more info.
1401+ -h, --help Display this help message.
1402+EOT
1403+}