Updated patches to work with the current trunk.
[rsync/rsync-patches.git] / db.diff
CommitLineData
cbdf862c
WD
1Added some DB-access routines to help rsync keep extra filesystem info
2about the files it is dealing with. This adds both the --db=CONFIG_FILE
3option and the "db config" daemon parameter.
4
5For the moment this only adds checksum caching when the --checksum option
bc3fcf1d 6is used. Future improvements may include:
cbdf862c
WD
7
8 - Updating of MD5 checksums when transferring any file, even w/o -c.
bc3fcf1d
WD
9 We should be able to extend this to work for MD4 checksums too if we
10 make the sender force checksum_seed to 0 when using a DB and having
11 the receiving side check to see if it got a 0 checksum_seed. (We
12 probably don't want to compute 2 MD4 checksums for the case where
13 the checksum_seed is non-zero.)
cbdf862c
WD
14
15 - Caching of path info that allows for the finding of files to use for
16 moving/linking/copying/alternate-basis-use.
17
18 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
19
20To use this patch, run these commands for a successful build:
21
cbdf862c
WD
22 patch -p1 <patches/db.diff
23 ./configure (optional if already run)
24 make
25
26diff --git a/Makefile.in b/Makefile.in
fc557362 27index feacb90..f9da8eb 100644
cbdf862c
WD
28--- a/Makefile.in
29+++ b/Makefile.in
fc557362 30@@ -36,7 +36,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
cbdf862c 31 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
fc557362 32 util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
cbdf862c
WD
33 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
34- fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
35+ fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
36 OBJS3=progress.o pipe.o
37 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
38 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
39diff --git a/checksum.c b/checksum.c
fc557362 40index 811b5b6..a7617af 100644
cbdf862c
WD
41--- a/checksum.c
42+++ b/checksum.c
43@@ -23,6 +23,7 @@
44
45 extern int checksum_seed;
46 extern int protocol_version;
47+extern int use_db;
48
fc557362
WD
49 /*
50 a simple 32 bit checksum that can be upadted from either end
51@@ -98,10 +99,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
cbdf862c
WD
52 }
53 }
54
55-void file_checksum(char *fname, char *sum, OFF_T size)
56+void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
57 {
58 struct map_struct *buf;
59- OFF_T i, len = size;
60+ OFF_T i, len = st_p->st_size;
61 md_context m;
62 int32 remainder;
63 int fd;
fc557362 64@@ -112,7 +113,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
cbdf862c
WD
65 if (fd == -1)
66 return;
67
68- buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
69+ buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
70
71 if (protocol_version >= 30) {
72 md5_begin(&m);
fc557362 73@@ -146,6 +147,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
cbdf862c
WD
74 mdfour_result(&m, (uchar *)sum);
75 }
76
77+ if (use_db)
78+ db_set_checksum(fname, st_p, sum);
79+
80 close(fd);
81 unmap_file(buf);
82 }
83diff --git a/cleanup.c b/cleanup.c
fc557362 84index 19ef072..ca46868 100644
cbdf862c
WD
85--- a/cleanup.c
86+++ b/cleanup.c
fc557362
WD
87@@ -25,6 +25,7 @@
88 extern int am_server;
89 extern int am_daemon;
cbdf862c 90 extern int io_error;
fc557362 91+extern int use_db;
cbdf862c
WD
92 extern int keep_partial;
93 extern int got_xfer_error;
fc557362
WD
94 extern int output_needs_newline;
95@@ -130,6 +131,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
cbdf862c
WD
96 /* FALLTHROUGH */
97 #include "case_N.h"
98
99+ if (use_db)
100+ db_disconnect();
101+
102+ /* FALLTHROUGH */
103+#include "case_N.h"
104+
105 if (cleanup_child_pid != -1) {
106 int status;
107 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
108diff --git a/clientserver.c b/clientserver.c
fc557362 109index b6afe00..37cad54 100644
cbdf862c
WD
110--- a/clientserver.c
111+++ b/clientserver.c
bc3fcf1d 112@@ -42,13 +42,16 @@ extern int numeric_ids;
cbdf862c
WD
113 extern int filesfrom_fd;
114 extern int remote_protocol;
115 extern int protocol_version;
116+extern int always_checksum;
117 extern int io_timeout;
118 extern int no_detach;
bc3fcf1d 119+extern int use_db;
cbdf862c 120 extern int write_batch;
bc3fcf1d 121 extern int default_af_hint;
cbdf862c
WD
122 extern int logfile_format_has_i;
123 extern int logfile_format_has_o_or_i;
124 extern mode_t orig_umask;
125+extern char *db_config;
126 extern char *bind_address;
cbdf862c 127 extern char *config_file;
fc557362
WD
128 extern char *logfile_format;
129@@ -648,6 +651,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
cbdf862c 130
bc3fcf1d
WD
131 log_init(1);
132
133+ if (*lp_db_config(i))
134+ db_read_config(FLOG, lp_db_config(i));
cbdf862c 135+
bc3fcf1d
WD
136 #ifdef HAVE_PUTENV
137 if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
138 char *modname, *modpath, *hostaddr, *hostname, *username;
fc557362 139@@ -856,6 +862,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
cbdf862c 140
bc3fcf1d
WD
141 am_server = 1; /* Don't let someone try to be tricky. */
142 quiet = 0;
143+ db_config = NULL;
144+ if (!always_checksum)
145+ use_db = 0;
146+
147 if (lp_ignore_errors(module_id))
148 ignore_errors = 1;
149 if (write_batch < 0)
cbdf862c 150diff --git a/configure.in b/configure.in
fc557362 151index bc7d4a7..43d51ff 100644
cbdf862c
WD
152--- a/configure.in
153+++ b/configure.in
fc557362 154@@ -312,7 +312,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
bc3fcf1d
WD
155 sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
156 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
157 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
158- popt.h popt/popt.h)
159+ popt.h popt/popt.h mysql/mysql.h sqlite3.h)
160 AC_HEADER_MAJOR
161
162 AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[
fc557362 163@@ -977,6 +977,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
cbdf862c
WD
164 fi
165 fi
166
bc3fcf1d
WD
167+AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
168+if test x$MYSQL_CONFIG = x1; then
169+ AC_MSG_CHECKING(for mysql version >= 4)
170+ mysql_version=`mysql_config --version`
171+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
172+ if test $mysql_major_version -lt 4; then
173+ AC_MSG_RESULT(no.. skipping MySQL)
174+ else
175+ AC_MSG_RESULT(yes)
176+
177+ MYSQL_CFLAGS=`mysql_config --cflags`
178+ MYSQL_LIBS=`mysql_config --libs`
179+
180+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
181+ LIBS="$MYSQL_LIBS $LIBS"
182+
183+ AC_CHECK_LIB(mysqlclient, mysql_init)
184+ fi
185+fi
186+
187+AC_CHECK_LIB(sqlite3, sqlite3_open)
188+AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
cbdf862c
WD
189+
190 case "$CC" in
191 ' checker'*|checker*)
192 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
193diff --git a/db.c b/db.c
194new file mode 100644
fc557362 195index 0000000..6855488
cbdf862c
WD
196--- /dev/null
197+++ b/db.c
bc3fcf1d 198@@ -0,0 +1,566 @@
cbdf862c
WD
199+/*
200+ * Routines to access extended file info via DB.
201+ *
202+ * Copyright (C) 2008 Wayne Davison
203+ *
204+ * This program is free software; you can redistribute it and/or modify
205+ * it under the terms of the GNU General Public License as published by
206+ * the Free Software Foundation; either version 3 of the License, or
207+ * (at your option) any later version.
208+ *
209+ * This program is distributed in the hope that it will be useful,
210+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
211+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
212+ * GNU General Public License for more details.
213+ *
214+ * You should have received a copy of the GNU General Public License along
215+ * with this program; if not, visit the http://fsf.org website.
216+ */
217+
218+#include "rsync.h"
219+#include "ifuncs.h"
220+
bc3fcf1d 221+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
cbdf862c 222+#define USE_MYSQL
cbdf862c
WD
223+#include <mysql/mysql.h>
224+#include <mysql/errmsg.h>
225+#endif
bc3fcf1d
WD
226+
227+#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
228+#define USE_SQLITE
cbdf862c 229+#include <sqlite3.h>
bc3fcf1d
WD
230+#ifndef HAVE_SQLITE3_OPEN_V2
231+#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
232+ sqlite3_open(dbname, dbhptr)
233+#endif
234+#ifndef HAVE_SQLITE3_PREPARE_V2
235+#define sqlite3_prepare_v2 sqlite3_prepare
236+#endif
cbdf862c
WD
237+#endif
238+
239+extern int protocol_version;
240+extern int checksum_len;
241+
242+#define DB_TYPE_NONE 0
243+#define DB_TYPE_MYSQL 1
244+#define DB_TYPE_SQLITE 2
245+
246+int use_db = DB_TYPE_NONE;
247+
248+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
249+static unsigned int dbport = 0;
250+
251+static union {
252+#ifdef USE_MYSQL
253+ MYSQL *mysql;
254+#endif
255+#ifdef USE_SQLITE
256+ sqlite3 *sqlite;
257+#endif
258+ void *all;
259+} dbh;
260+
261+#define SEL_DEV 0
262+#define SEL_SUM 1
263+#define REP_SUM 2
264+#define MAX_PREP_CNT 3
265+
266+static union {
267+#ifdef USE_MYSQL
268+ MYSQL_STMT *mysql;
269+#endif
270+#ifdef USE_SQLITE
271+ sqlite3_stmt *sqlite;
272+#endif
273+ void *all;
274+} statements[MAX_PREP_CNT];
275+
276+static int md_num;
277+static enum logcode log_code;
278+
bc3fcf1d 279+#ifdef USE_MYSQL
cbdf862c
WD
280+static unsigned int bind_disk_id;
281+static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
bc3fcf1d
WD
282+static char bind_sum[MAX_DIGEST_LEN];
283+#endif
284+static char bind_thishost[256];
cbdf862c
WD
285+static int bind_thishost_len;
286+
287+static unsigned int prior_disk_id = 0;
288+static unsigned long long prior_devno = 0;
289+
290+int db_read_config(enum logcode code, const char *config_file)
291+{
292+ char buf[2048], *cp;
293+ FILE *fp;
294+ int lineno = 0;
295+
296+ log_code = code;
297+
298+ bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
299+
300+ if (!(fp = fopen(config_file, "r"))) {
301+ rsyserr(log_code, errno, "unable to open %s", config_file);
302+ return 0;
303+ }
304+ while (fgets(buf, sizeof buf, fp)) {
305+ lineno++;
306+ if ((cp = strchr(buf, '#')) == NULL
307+ && (cp = strchr(buf, '\r')) == NULL
308+ && (cp = strchr(buf, '\n')) == NULL)
309+ cp = buf + strlen(buf);
310+ while (cp != buf && isSpace(cp-1)) cp--;
311+ *cp = '\0';
312+
313+ if (!*buf)
314+ continue;
315+
316+ if (!(cp = strchr(buf, ':')))
317+ goto invalid_line;
318+ *cp++ = '\0';
319+
320+ while (isSpace(cp)) cp++;
321+ if (strcasecmp(buf, "dbhost") == 0)
322+ dbhost = strdup(cp);
323+ else if (strcasecmp(buf, "dbuser") == 0)
324+ dbuser = strdup(cp);
325+ else if (strcasecmp(buf, "dbpass") == 0)
326+ dbpass = strdup(cp);
327+ else if (strcasecmp(buf, "dbname") == 0)
328+ dbname = strdup(cp);
329+ else if (strcasecmp(buf, "dbport") == 0)
330+ dbport = atoi(cp);
331+ else if (strcasecmp(buf, "thishost") == 0)
332+ bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
333+ else if (strcasecmp(buf, "dbtype") == 0) {
334+#ifdef USE_MYSQL
335+ if (strcasecmp(cp, "mysql") == 0) {
336+ use_db = DB_TYPE_MYSQL;
337+ continue;
338+ }
339+#endif
340+#ifdef USE_SQLITE
341+ if (strcasecmp(cp, "sqlite") == 0) {
342+ use_db = DB_TYPE_SQLITE;
343+ continue;
344+ }
345+#endif
346+ rprintf(log_code,
347+ "Unsupported dbtype on line #%d in %s.\n",
348+ lineno, config_file);
349+ use_db = DB_TYPE_NONE;
350+ return 0;
351+ } else {
352+ invalid_line:
353+ rprintf(log_code, "Invalid line #%d in %s\n",
354+ lineno, config_file);
355+ use_db = DB_TYPE_NONE;
356+ return 0;
357+ }
358+ }
359+ fclose(fp);
360+
361+ if (bind_thishost_len >= (int)sizeof bind_thishost)
362+ bind_thishost_len = sizeof bind_thishost - 1;
363+
364+ if (!use_db || !dbname) {
365+ rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
366+ use_db = DB_TYPE_NONE;
367+ return 0;
368+ }
369+
370+ md_num = protocol_version >= 30 ? 5 : 4;
371+
372+ return 1;
373+}
374+
375+#ifdef USE_MYSQL
376+static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
377+{
378+ va_list ap;
379+ char *query;
380+ int qlen, param_cnt;
381+ MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
382+
383+ if (stmt == NULL)
384+ out_of_memory("prepare_mysql");
385+
386+ va_start(ap, fmt);
387+ qlen = vasprintf(&query, fmt, ap);
388+ va_end(ap);
389+ if (qlen < 0)
390+ out_of_memory("prepare_mysql");
391+
392+ if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
393+ rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt));
394+ return NULL;
395+ }
396+ free(query);
397+
398+ if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
399+ rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n",
400+ param_cnt, bind_cnt);
401+ return NULL;
402+ }
403+ if (bind_cnt)
404+ mysql_stmt_bind_param(stmt, binds);
405+
406+ return stmt;
407+}
408+#endif
409+
410+#ifdef USE_MYSQL
411+static int db_connect_mysql(void)
412+{
413+ MYSQL_BIND binds[10];
414+
415+ if (!(dbh.mysql = mysql_init(NULL)))
416+ out_of_memory("db_read_config");
417+
418+ if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0))
419+ return 0;
420+
421+ memset(binds, 0, sizeof binds);
422+ binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
423+ binds[0].buffer = &bind_devno;
424+ binds[1].buffer_type = MYSQL_TYPE_STRING;
425+ binds[1].buffer = &bind_thishost;
426+ binds[1].buffer_length = bind_thishost_len;
427+ statements[SEL_DEV].mysql = prepare_mysql(binds, 2,
428+ "SELECT disk_id"
429+ " FROM disk"
430+ " WHERE devno = ? AND host = ? AND mounted = 1");
431+ if (!statements[SEL_DEV].mysql)
432+ return 0;
433+
434+ memset(binds, 0, sizeof binds);
435+ binds[0].buffer_type = MYSQL_TYPE_LONG;
436+ binds[0].buffer = &bind_disk_id;
437+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
438+ binds[1].buffer = &bind_ino;
439+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
440+ binds[2].buffer = &bind_size;
441+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
442+ binds[3].buffer = &bind_mtime;
443+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
444+ binds[4].buffer = &bind_ctime;
445+ statements[SEL_SUM].mysql = prepare_mysql(binds, 5,
446+ "SELECT checksum"
447+ " FROM inode_map"
448+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
449+ " AND size = ? AND mtime = ? AND ctime = ?",
450+ md_num);
451+ if (!statements[SEL_SUM].mysql)
452+ return 0;
453+
454+ memset(binds, 0, sizeof binds);
455+ binds[0].buffer_type = MYSQL_TYPE_LONG;
456+ binds[0].buffer = &bind_disk_id;
457+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
458+ binds[1].buffer = &bind_ino;
459+ binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG;
460+ binds[2].buffer = binds[6].buffer = &bind_size;
461+ binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG;
462+ binds[3].buffer = binds[7].buffer = &bind_mtime;
463+ binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG;
464+ binds[4].buffer = binds[8].buffer = &bind_ctime;
465+ binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB;
466+ binds[5].buffer = binds[9].buffer = &bind_sum;
467+ binds[5].buffer_length = binds[9].buffer_length = checksum_len;
468+ statements[REP_SUM].mysql = prepare_mysql(binds, 10,
469+ "INSERT INTO inode_map"
470+ " SET disk_id = ?, ino = ?, sum_type = %d,"
471+ " size = ?, mtime = ?, ctime = ?, checksum = ?"
472+ " ON DUPLICATE KEY"
473+ " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?",
474+ md_num, md_num);
475+ if (!statements[REP_SUM].mysql)
476+ return 0;
477+
478+ return 1;
479+}
480+#endif
481+
482+#ifdef USE_SQLITE
483+static int db_connect_sqlite(void)
484+{
485+ char *sql;
486+
cbdf862c
WD
487+ if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0)
488+ return 0;
cbdf862c
WD
489+
490+ sql = "SELECT disk_id"
491+ " FROM disk"
492+ " WHERE devno = ? AND host = ? AND mounted = 1";
493+ if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0)
494+ return 0;
495+
496+ if (asprintf(&sql,
497+ "SELECT checksum"
498+ " FROM inode_map"
499+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
500+ " AND size = ? AND mtime = ? AND ctime = ?",
501+ md_num) < 0
502+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0)
503+ return 0;
504+ free(sql);
505+
506+ if (asprintf(&sql,
507+ "INSERT OR REPLACE INTO inode_map"
508+ " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
509+ " VALUES(?, ?, %d, ?, ?, ?, ?)",
510+ md_num) < 0
511+ || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0)
512+ return 0;
513+ free(sql);
514+
515+ return 1;
516+}
517+#endif
518+
519+int db_connect(void)
520+{
521+ switch (use_db) {
522+#ifdef USE_MYSQL
523+ case DB_TYPE_MYSQL:
524+ if (db_connect_mysql())
525+ return 1;
526+ break;
527+#endif
528+#ifdef USE_SQLITE
529+ case DB_TYPE_SQLITE:
530+ if (db_connect_sqlite())
531+ return 1;
532+ break;
533+#endif
534+ }
535+
536+ rprintf(log_code, "Unable to connect to DB\n");
537+ db_disconnect();
538+ use_db = DB_TYPE_NONE;
539+
540+ return 0;
541+}
542+
543+void db_disconnect(void)
544+{
545+ int ndx;
546+
547+ if (!dbh.all)
548+ return;
549+
550+ for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
551+ if (statements[ndx].all) {
552+ switch (use_db) {
553+#ifdef USE_MYSQL
554+ case DB_TYPE_MYSQL:
555+ mysql_stmt_close(statements[ndx].mysql);
556+ break;
557+#endif
558+#ifdef USE_SQLITE
559+ case DB_TYPE_SQLITE:
560+ sqlite3_finalize(statements[ndx].sqlite);
561+ break;
562+#endif
563+ }
564+ statements[ndx].all = NULL;
565+ }
566+ }
567+
568+ switch (use_db) {
569+#ifdef USE_MYSQL
570+ case DB_TYPE_MYSQL:
571+ mysql_close(dbh.mysql);
572+ break;
573+#endif
574+#ifdef USE_SQLITE
575+ case DB_TYPE_SQLITE:
576+ sqlite3_close(dbh.sqlite);
577+ break;
578+#endif
579+ }
580+
581+ dbh.all = NULL;
582+}
583+
bc3fcf1d 584+#ifdef USE_MYSQL
cbdf862c
WD
585+static MYSQL_STMT *exec_mysql(int ndx)
586+{
587+ MYSQL_STMT *stmt = statements[ndx].mysql;
588+ int rc;
589+
590+ if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
591+ db_disconnect();
592+ if (db_connect()) {
593+ stmt = statements[ndx].mysql;
594+ rc = mysql_stmt_execute(stmt);
595+ }
596+ }
597+ if (rc != 0) {
598+ rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
599+ return NULL;
600+ }
601+
602+ return stmt;
603+}
bc3fcf1d 604+#endif
cbdf862c 605+
bc3fcf1d 606+#ifdef USE_MYSQL
cbdf862c
WD
607+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx)
608+{
609+ unsigned long length[32];
610+ my_bool is_null[32], error[32];
611+ MYSQL_STMT *stmt;
612+ int i, rc;
613+
614+ if (bind_cnt > 32)
615+ exit_cleanup(RERR_UNSUPPORTED);
616+
617+ if ((stmt = exec_mysql(ndx)) == NULL)
618+ return 0;
619+
620+ for (i = 0; i < bind_cnt; i++) {
621+ binds[i].is_null = &is_null[i];
622+ binds[i].length = &length[i];
623+ binds[i].error = &error[i];
624+ }
625+ mysql_stmt_bind_result(stmt, binds);
626+
627+ if ((rc = mysql_stmt_fetch(stmt)) != 0) {
628+ if (rc != MYSQL_NO_DATA) {
629+ rprintf(log_code, "SELECT fetch failed: %s\n",
630+ mysql_stmt_error(stmt));
631+ }
632+ mysql_stmt_free_result(stmt);
633+ return 0;
634+ }
635+
636+ mysql_stmt_free_result(stmt);
637+
638+ return is_null[0] ? 0 : 1;
639+}
bc3fcf1d 640+#endif
cbdf862c
WD
641+
642+static void get_disk_id(unsigned long long devno)
643+{
644+ switch (use_db) {
645+#ifdef USE_MYSQL
646+ case DB_TYPE_MYSQL: {
647+ MYSQL_BIND binds[1];
648+
649+ bind_devno = devno; /* The one variable SEL_DEV input value. */
650+
651+ /* Bind where to put the output. */
652+ binds[0].buffer_type = MYSQL_TYPE_LONG;
653+ binds[0].buffer = &prior_disk_id;
654+ if (!fetch_mysql(binds, 1, SEL_DEV))
655+ prior_disk_id = 0;
656+ break;
657+ }
658+#endif
659+#ifdef USE_SQLITE
660+ case DB_TYPE_SQLITE: {
661+ sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
662+ sqlite3_bind_int64(stmt, 1, devno);
663+ sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC);
664+ if (sqlite3_step(stmt) == SQLITE_ROW)
665+ prior_disk_id = sqlite3_column_int(stmt, 0);
666+ else
667+ prior_disk_id = 0;
668+ sqlite3_reset(stmt);
669+ break;
670+ }
671+#endif
672+ }
673+
674+ prior_devno = devno;
675+}
676+
677+int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum)
678+{
679+ if (prior_devno != st_p->st_dev)
680+ get_disk_id(st_p->st_dev);
681+ if (prior_disk_id == 0)
682+ return 0;
683+
684+ switch (use_db) {
685+#ifdef USE_MYSQL
686+ case DB_TYPE_MYSQL: {
687+ MYSQL_BIND binds[1];
688+
689+ bind_disk_id = prior_disk_id;
690+ bind_ino = st_p->st_ino;
691+ bind_size = st_p->st_size;
692+ bind_mtime = st_p->st_mtime;
693+ bind_ctime = st_p->st_ctime;
694+
695+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
696+ binds[0].buffer = sum;
697+ binds[0].buffer_length = checksum_len;
698+ return fetch_mysql(binds, 1, SEL_SUM);
699+ }
700+#endif
701+#ifdef USE_SQLITE
702+ case DB_TYPE_SQLITE: {
703+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
704+ sqlite3_bind_int(stmt, 1, prior_disk_id);
705+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
706+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
707+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
708+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
709+ if (sqlite3_step(stmt) == SQLITE_ROW) {
710+ int len = sqlite3_column_bytes(stmt, 0);
711+ if (len > MAX_DIGEST_LEN)
712+ len = MAX_DIGEST_LEN;
713+ memcpy(sum, sqlite3_column_blob(stmt, 0), len);
714+ sqlite3_reset(stmt);
715+ return 1;
716+ }
717+ sqlite3_reset(stmt);
718+ return 0;
719+ }
720+#endif
721+ }
722+
723+ return 0;
724+}
725+
726+int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum)
727+{
728+ if (prior_devno != st_p->st_dev)
729+ get_disk_id(st_p->st_dev);
730+ if (prior_disk_id == 0)
731+ return 0;
732+
733+ switch (use_db) {
734+#ifdef USE_MYSQL
735+ case DB_TYPE_MYSQL: {
736+ bind_disk_id = prior_disk_id;
737+ bind_ino = st_p->st_ino;
738+ bind_size = st_p->st_size;
739+ bind_mtime = st_p->st_mtime;
740+ bind_ctime = st_p->st_ctime;
741+ memcpy(bind_sum, sum, checksum_len);
742+
743+ return exec_mysql(REP_SUM) != NULL;
744+ }
745+#endif
746+#ifdef USE_SQLITE
747+ case DB_TYPE_SQLITE: {
748+ int rc;
749+ sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
750+ sqlite3_bind_int(stmt, 1, prior_disk_id);
751+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
752+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
753+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
754+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
755+ sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT);
756+ rc = sqlite3_step(stmt);
757+ sqlite3_reset(stmt);
758+ return rc == SQLITE_DONE;
759+ }
760+#endif
761+ }
762+
763+ return 0;
764+}
765diff --git a/flist.c b/flist.c
fc557362 766index 09b4fc5..8d280e9 100644
cbdf862c
WD
767--- a/flist.c
768+++ b/flist.c
fc557362
WD
769@@ -54,6 +54,7 @@ extern int preserve_specials;
770 extern int missing_args;
cbdf862c
WD
771 extern int uid_ndx;
772 extern int gid_ndx;
773+extern int use_db;
774 extern int eol_nulls;
775 extern int relative_paths;
776 extern int implied_dirs;
fc557362
WD
777@@ -1267,11 +1268,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
778 extra_len += EXTRA_LEN;
cbdf862c
WD
779 #endif
780
fc557362 781- if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
cbdf862c 782- file_checksum(thisname, tmp_sum, st.st_size);
fc557362
WD
783- if (sender_keeps_checksum)
784- extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
785- }
786+ if (sender_keeps_checksum && S_ISREG(st.st_mode))
787+ extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
cbdf862c 788
fc557362
WD
789 #if EXTRA_ROUNDING > 0
790 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
791@@ -1347,8 +1345,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
792 return NULL;
793 }
794
795- if (sender_keeps_checksum && S_ISREG(st.st_mode))
796- memcpy(F_SUM(file), tmp_sum, checksum_len);
cbdf862c
WD
797+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
798+ if (!use_db || !db_get_checksum(thisname, &st, tmp_sum))
799+ file_checksum(thisname, &st, tmp_sum);
fc557362
WD
800+ if (sender_keeps_checksum)
801+ memcpy(F_SUM(file), tmp_sum, checksum_len);
cbdf862c 802+ }
fc557362
WD
803
804 if (unsort_ndx)
805 F_NDX(file) = stats.num_dirs;
806@@ -2010,6 +2012,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
cbdf862c
WD
807 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
808 int implied_dot_dir = 0;
809
810+ if (use_db)
811+ db_connect();
812+
813 rprintf(FLOG, "building file list\n");
814 if (show_filelist_p())
815 start_filelist_progress("building file list");
816diff --git a/generator.c b/generator.c
fc557362 817index 12007a1..315463f 100644
cbdf862c
WD
818--- a/generator.c
819+++ b/generator.c
fc557362 820@@ -60,6 +60,7 @@ extern int human_readable;
cbdf862c
WD
821 extern int ignore_existing;
822 extern int ignore_non_existing;
823 extern int inplace;
824+extern int use_db;
825 extern int append_mode;
826 extern int make_backups;
827 extern int csum_length;
fc557362 828@@ -531,7 +532,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
cbdf862c
WD
829 of the file time to determine whether to sync */
830 if (always_checksum > 0 && S_ISREG(st->st_mode)) {
831 char sum[MAX_DIGEST_LEN];
832- file_checksum(fn, sum, st->st_size);
833+ if (!use_db || !db_get_checksum(fn, st, sum))
834+ file_checksum(fn, st, sum);
835 return memcmp(sum, F_SUM(file), checksum_len) == 0;
836 }
837
fc557362 838@@ -2022,6 +2024,9 @@ void generate_files(int f_out, const char *local_name)
cbdf862c
WD
839 : "enabled");
840 }
841
842+ if (use_db && always_checksum)
843+ db_connect();
844+
845 /* Since we often fill up the outgoing socket and then just sit around
846 * waiting for the other 2 processes to do their thing, we don't want
847 * to exit on a timeout. If the data stops flowing, the receiver will
848diff --git a/loadparm.c b/loadparm.c
fc557362 849index 8e48e6d..e6eaec8 100644
cbdf862c
WD
850--- a/loadparm.c
851+++ b/loadparm.c
fc557362 852@@ -107,6 +107,7 @@ typedef struct {
cbdf862c
WD
853 char *auth_users;
854 char *charset;
855 char *comment;
856+ char *db_config;
857 char *dont_compress;
858 char *exclude;
859 char *exclude_from;
fc557362 860@@ -181,6 +182,7 @@ static const all_vars Defaults = {
cbdf862c
WD
861 /* auth_users; */ NULL,
862 /* charset; */ NULL,
863 /* comment; */ NULL,
864+ /* db_config; */ NULL,
865 /* dont_compress; */ DEFAULT_DONT_COMPRESS,
866 /* exclude; */ NULL,
867 /* exclude_from; */ NULL,
fc557362
WD
868@@ -316,6 +318,7 @@ static struct parm_struct parm_table[] =
869 {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0},
870 {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0},
871 {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0},
872+ {"db config", P_STRING, P_LOCAL, &Vars.l.db_config, NULL,0},
873 {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0},
874 {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0},
875 {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0},
876@@ -396,6 +399,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
cbdf862c
WD
877 FN_LOCAL_STRING(lp_auth_users, auth_users)
878 FN_LOCAL_STRING(lp_charset, charset)
879 FN_LOCAL_STRING(lp_comment, comment)
880+FN_LOCAL_STRING(lp_db_config, db_config)
881 FN_LOCAL_STRING(lp_dont_compress, dont_compress)
882 FN_LOCAL_STRING(lp_exclude, exclude)
883 FN_LOCAL_STRING(lp_exclude_from, exclude_from)
884diff --git a/main.c b/main.c
fc557362 885index 2ef2f47..ac88e12 100644
cbdf862c
WD
886--- a/main.c
887+++ b/main.c
888@@ -49,6 +49,7 @@ extern int copy_unsafe_links;
889 extern int keep_dirlinks;
890 extern int preserve_hard_links;
891 extern int protocol_version;
892+extern int always_checksum;
893 extern int file_total;
894 extern int recurse;
895 extern int xfer_dirs;
fc557362
WD
896@@ -74,6 +75,7 @@ extern char *filesfrom_host;
897 extern char *partial_dir;
cbdf862c 898 extern char *dest_option;
cbdf862c
WD
899 extern char *rsync_path;
900+extern char *db_config;
901 extern char *shell_cmd;
902 extern char *batch_name;
903 extern char *password_file;
fc557362 904@@ -1565,6 +1567,9 @@ int main(int argc,char *argv[])
cbdf862c
WD
905 exit_cleanup(RERR_SYNTAX);
906 }
907
bc3fcf1d 908+ if (db_config && always_checksum)
cbdf862c
WD
909+ db_read_config(FERROR, db_config);
910+
911 if (am_server) {
912 set_nonblocking(STDIN_FILENO);
913 set_nonblocking(STDOUT_FILENO);
914diff --git a/options.c b/options.c
fc557362 915index e7c6c61..d47cb7c 100644
cbdf862c
WD
916--- a/options.c
917+++ b/options.c
918@@ -92,6 +92,7 @@ int use_qsort = 0;
919 char *files_from = NULL;
920 int filesfrom_fd = -1;
921 char *filesfrom_host = NULL;
922+char *db_config = NULL;
923 int eol_nulls = 0;
924 int protect_args = 0;
fc557362
WD
925 int human_readable = 1;
926@@ -566,6 +567,7 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
927 char const *links = "no ";
928 char const *iconv = "no ";
929 char const *ipv6 = "no ";
930+ char const *db = "no ";
931 STRUCT_STAT *dumstat;
932
933 #if SUBPROTOCOL_VERSION != 0
fc557362 934@@ -599,6 +601,11 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
935 #if defined HAVE_LUTIMES && defined HAVE_UTIMES
936 symtimes = "";
937 #endif
938+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
939+ db = "";
940+#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
941+ db = "";
942+#endif
943
944 rprintf(f, "%s version %s protocol version %d%s\n",
945 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
fc557362 946@@ -612,8 +619,8 @@ static void print_rsync_version(enum logcode f)
bc3fcf1d
WD
947 (int)(sizeof (int64) * 8));
948 rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
949 got_socketpair, hardlinks, links, ipv6, have_inplace);
950- rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n",
951- have_inplace, acls, xattrs, iconv, symtimes);
952+ rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n",
953+ have_inplace, acls, xattrs, iconv, symtimes, db);
954
955 #ifdef MAINTAINER_MODE
956 rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
fc557362 957@@ -661,6 +668,7 @@ void usage(enum logcode F)
cbdf862c
WD
958 rprintf(F," -q, --quiet suppress non-error messages\n");
959 rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n");
960 rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n");
bc3fcf1d 961+ rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n");
cbdf862c
WD
962 rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n");
963 rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n");
964 rprintf(F," -r, --recursive recurse into directories\n");
fc557362 965@@ -933,6 +941,7 @@ static struct poptOption long_options[] = {
cbdf862c
WD
966 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
967 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
968 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
969+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
970 {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
971 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
972 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
973diff --git a/pipe.c b/pipe.c
fc557362 974index a33117c..2f6aa21 100644
cbdf862c
WD
975--- a/pipe.c
976+++ b/pipe.c
fc557362 977@@ -27,6 +27,9 @@ extern int am_server;
cbdf862c
WD
978 extern int blocking_io;
979 extern int filesfrom_fd;
fc557362 980 extern int munge_symlinks;
cbdf862c 981+extern int always_checksum;
cbdf862c
WD
982+extern int use_db;
983+extern char *db_config;
984 extern mode_t orig_umask;
985 extern char *logfile_name;
986 extern int remote_option_cnt;
fc557362 987@@ -143,6 +146,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
988 logfile_close();
989 }
990
991+ use_db = 0;
992+ db_config = NULL;
993+
994 if (remote_option_cnt) {
995 int rc = remote_option_cnt + 1;
996 const char **rv = remote_options;
fc557362 997@@ -150,6 +156,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
cbdf862c
WD
998 option_error();
999 exit_cleanup(RERR_SYNTAX);
1000 }
bc3fcf1d 1001+ if (db_config && always_checksum)
cbdf862c
WD
1002+ db_read_config(FERROR, db_config);
1003 }
1004
1005 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
bc3fcf1d 1006diff --git a/rsync.yo b/rsync.yo
fc557362 1007index 941f7a5..1b81189 100644
bc3fcf1d
WD
1008--- a/rsync.yo
1009+++ b/rsync.yo
fc557362 1010@@ -323,6 +323,7 @@ to the detailed description below for a complete description. verb(
bc3fcf1d
WD
1011 -q, --quiet suppress non-error messages
1012 --no-motd suppress daemon-mode MOTD (see caveat)
1013 -c, --checksum skip based on checksum, not mod-time & size
1014+ --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
1015 -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)
1016 --no-OPTION turn off an implied OPTION (e.g. --no-D)
1017 -r, --recursive recurse into directories
fc557362 1018@@ -584,6 +585,47 @@ checksum that is generated as the file is transferred, but that
bc3fcf1d
WD
1019 automatic after-the-transfer verification has nothing to do with this
1020 option's before-the-transfer "Does this file need to be updated?" check.
cbdf862c 1021
bc3fcf1d
WD
1022+dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read
1023+that holds connection details for a database of checksum information.
1024+When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
1025+use cached checksum information from the DB, and will update it if it is
1026+missing.
1027+
1028+The currently supported DB choices are MySQL and SQLite. For example, a
1029+MySQL configuration might look like this:
1030+
1031+verb( dbtype: mysql
1032+ dbhost: 127.0.0.1
1033+ dbname: rsyncdb
1034+ dbuser: rsyncuser
1035+ dbpass: somepass
1036+ port: 3306
1037+ thishost: hostname )
1038+
1039+And a SQLite configuration might look like this:
1040+
1041+verb( dbtype: SQLite
1042+ dbname: /var/cache/rsync/sum.db )
1043+
1044+This option only affects one side of a transfer. See the
1045+bf(--remote-option) option for a way to specify the option for both
1046+sides of the transfer (with each side reading the config file from
1047+their local filesystem). For example:
1048+
1049+verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
1050+
1051+See the perl script "rsyncdb" in the support directory of the source code
1052+(which may also be installed in /usr/bin) for a way to create the tables,
1053+populate the mounted-disk information, check files against their checksums,
1054+and update both the MD4 and MD5 checksums for files at the same time (since
1055+an rsync copy will only update one or the other).
1056+
1057+You can use a single MySQL DB for all your hosts if you give each one
1058+their own "thishost" name and setup their device-mapping data. Or feel
1059+free to use separate databases, separate servers, etc. See the rsync
1060+daemon's "db config" parameter for how to configure a daemon to use a DB
1061+(since a client cannot control this parameter on a daemon).
1062+
1063 dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
1064 way of saying you want recursion and want to preserve almost
1065 everything (with -H being a notable omission).
1066diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
fc557362 1067index d4978cd..68c588c 100644
bc3fcf1d
WD
1068--- a/rsyncd.conf.yo
1069+++ b/rsyncd.conf.yo
fc557362 1070@@ -281,6 +281,18 @@ is daemon. This setting has no effect if the "log file" setting is a
bc3fcf1d
WD
1071 non-empty string (either set in the per-modules settings, or inherited
1072 from the global settings).
cbdf862c 1073
bc3fcf1d
WD
1074+dit(bf(db config)) This parameter specifies a config file to read that
1075+holds connection details for a database of checksum information.
1076+
1077+The config file will be read-in prior to any chroot restrictions, but
1078+the connection occurs from inside the chroot. This means that you
1079+should use a socket connection (e.g. 127.0.0.1 rather than localhost)
1080+for a MySQL config from inside a chroot. For SQLite, the DB file must
1081+be placed inside the chroot (though it can be placed outside the
1082+transfer dir if you configured an inside-chroot path).
1083+
1084+See the bf(--db=CONFIG_FILE) option for full details.
1085+
e66d6d51 1086 dit(bf(max verbosity)) This parameter allows you to control
bc3fcf1d
WD
1087 the maximum amount of verbose information that you'll allow the daemon to
1088 generate (since the information goes into the log file). The default is 1,
1089diff --git a/support/rsyncdb b/support/rsyncdb
cbdf862c 1090new file mode 100755
fc557362 1091index 0000000..801068c
cbdf862c 1092--- /dev/null
bc3fcf1d 1093+++ b/support/rsyncdb
fc557362
WD
1094@@ -0,0 +1,331 @@
1095+#!/usr/bin/perl -w
cbdf862c
WD
1096+use strict;
1097+
1098+use DBI;
1099+use Getopt::Long;
1100+use Cwd qw(abs_path cwd);
1101+use Digest::MD4;
1102+use Digest::MD5;
1103+
1104+my $MOUNT_FILE = '/etc/mtab';
1105+
1106+&Getopt::Long::Configure('bundling');
1107+&usage if !&GetOptions(
1108+ 'db=s' => \( my $db_config ),
bc3fcf1d 1109+ 'init' => \( my $init_db ),
cbdf862c
WD
1110+ 'mounts|m' => \( my $update_mounts ),
1111+ 'recurse|r' => \( my $recurse_opt ),
1112+ 'check|c' => \( my $check_opt ),
1113+ 'verbose|v+' => \( my $verbosity = 0 ),
1114+ 'help|h' => \( my $help_opt ),
1115+);
1116+&usage if $help_opt || !defined $db_config;
1117+
1118+my %config;
1119+open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n";
1120+while (<IN>) {
1121+ s/[#\r\n].*//s;
1122+ next if /^$/;
1123+ my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n";
1124+ $config{$key} = $val;
1125+}
1126+close IN;
1127+
1128+die "You must define at least dbtype and dbname in $db_config\n"
1129+ unless defined $config{'dbtype'} && defined $config{'dbname'};
1130+
bc3fcf1d
WD
1131+my $sqlite = $config{'dbtype'} =~ /^sqlite$/i;
1132+
cbdf862c
WD
1133+my $thishost = $config{'thishost'} || 'localhost';
1134+
bc3fcf1d
WD
1135+my $connect = 'DBI:' . $config{'dbtype'} . ':';
1136+$connect .= 'dbname=' . $config{'dbname'} if $sqlite;
1137+$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db;
cbdf862c
WD
1138+$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'};
1139+$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'};
1140+
1141+my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'})
1142+ or die "DB connection failed\n";
1143+
1144+END {
1145+ $dbh->disconnect if defined $dbh;
1146+}
1147+
bc3fcf1d
WD
1148+if ($init_db) {
1149+ my $unsigned = $sqlite ? '' : 'unsigned';
1150+ my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT';
1151+ my $dbname = $config{'dbname'};
1152+
1153+ if (!$sqlite) {
1154+ $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`");
1155+ $dbh->do("USE `$dbname`");
1156+ }
1157+
1158+ print "Dropping old tables (if they exist) ...\n" if $verbosity;
1159+ $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr;
1160+ $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr;
1161+
1162+ print "Creating empty tables ...\n" if $verbosity;
1163+ $dbh->do("
1164+ CREATE TABLE disk (
1165+ disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment,
1166+ devno bigint $unsigned NOT NULL,
1167+ host varchar(256) NOT NULL default 'localhost',
1168+ mounted tinyint NOT NULL default '1',
1169+ comment varchar(256) default NULL
1170+ )") or die $dbh->errstr;
1171+
1172+ $dbh->do("
1173+ CREATE TABLE inode_map (
1174+ disk_id integer $unsigned NOT NULL,
1175+ ino bigint $unsigned NOT NULL,
1176+ size bigint $unsigned NOT NULL,
1177+ mtime bigint NOT NULL,
1178+ ctime bigint NOT NULL,
1179+ sum_type tinyint NOT NULL default '0',
1180+ checksum binary(16) NOT NULL,
1181+ PRIMARY KEY (disk_id,ino,sum_type)
1182+ )") or die $dbh->errstr;
1183+
1184+ exit unless $update_mounts;
1185+}
1186+
cbdf862c
WD
1187+my $sel_disk_H = $dbh->prepare("
1188+ SELECT disk_id, devno, mounted, comment
1189+ FROM disk
1190+ WHERE host = ?
1191+ ") or die $dbh->errstr;
1192+
1193+my $ins_disk_H = $dbh->prepare("
1194+ INSERT INTO disk
1195+ (devno, host, mounted, comment)
1196+ VALUES(?, ?, ?, ?)
1197+ ") or die $dbh->errstr;
1198+
1199+my $up_disk_H = $dbh->prepare("
1200+ UPDATE disk
1201+ SET mounted = ?
1202+ WHERE disk_id = ?
1203+ ") or die $dbh->errstr;
1204+
bc3fcf1d 1205+my $row_id = $sqlite ? 'ROWID' : 'ID';
cbdf862c
WD
1206+my $sel_lastid_H = $dbh->prepare("
1207+ SELECT LAST_INSERT_$row_id()
1208+ ") or die $dbh->errstr;
1209+
1210+my $sel_sum_H = $dbh->prepare("
1211+ SELECT sum_type, checksum
1212+ FROM inode_map
1213+ WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ?
1214+ ") or die $dbh->errstr;
1215+
1216+my $rep_sum_H = $dbh->prepare("
1217+ REPLACE INTO inode_map
1218+ (disk_id, ino, size, mtime, ctime, sum_type, checksum)
1219+ VALUES(?, ?, ?, ?, ?, ?, ?)
1220+ ") or die $dbh->errstr;
1221+
1222+my %mounts;
1223+if ($update_mounts) {
1224+ open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n";
1225+ while (<IN>) {
1226+ my($devname, $mnt) = (split)[0,1];
1227+ next unless $devname =~ m#^/dev#;
1228+ my($devno) = (stat($mnt))[0];
1229+ if (!defined $devno) {
1230+ warn "Unable to stat $mnt: $!\n";
1231+ next;
1232+ }
1233+ $mounts{$devno} = "$devname on $mnt";
1234+ }
1235+ close IN;
1236+}
1237+
1238+my %disk_id;
1239+$sel_disk_H->execute($thishost);
1240+while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) {
1241+ if ($update_mounts) {
1242+ if (defined $mounts{$devno}) {
1243+ if ($comment ne $mounts{$devno}) {
1244+ if ($mounted) {
bc3fcf1d 1245+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1246+ $up_disk_H->execute(0, $disk_id);
1247+ }
1248+ next;
1249+ }
1250+ if (!$mounted) {
bc3fcf1d 1251+ print "Mounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1252+ $up_disk_H->execute(1, $disk_id);
1253+ }
1254+ } else {
1255+ if ($mounted) {
bc3fcf1d 1256+ print "Umounting $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1257+ $up_disk_H->execute(0, $disk_id);
1258+ }
1259+ next;
1260+ }
1261+ } else {
1262+ next unless $mounted;
1263+ }
1264+ $disk_id{$devno} = $disk_id;
1265+}
1266+$sel_disk_H->finish;
1267+
1268+if ($update_mounts) {
1269+ while (my($devno, $comment) = each %mounts) {
1270+ next if $disk_id{$devno};
bc3fcf1d 1271+ print "Adding $comment ($thishost:$devno)\n" if $verbosity;
cbdf862c
WD
1272+ $ins_disk_H->execute($devno, $thishost, 1, $comment);
1273+ $sel_lastid_H->execute;
1274+ ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array;
1275+ $sel_lastid_H->finish;
1276+ }
bc3fcf1d 1277+ exit;
cbdf862c
WD
1278+}
1279+
1280+my $start_dir = cwd();
1281+
1282+my @dirs = @ARGV;
1283+@dirs = '.' unless @dirs;
1284+foreach (@dirs) {
1285+ $_ = abs_path($_);
1286+}
1287+
1288+$| = 1;
1289+
1290+my $exit_code = 0;
1291+
1292+my $md4 = Digest::MD4->new;
1293+my $md5 = Digest::MD5->new;
1294+
1295+while (@dirs) {
1296+ my $dir = shift @dirs;
1297+
1298+ if (!chdir($dir)) {
1299+ warn "Unable to chdir to $dir: $!\n";
1300+ next;
1301+ }
1302+ if (!opendir(DP, '.')) {
1303+ warn "Unable to opendir $dir: $!\n";
1304+ next;
1305+ }
1306+
1307+ my $reldir = $dir;
1308+ $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
1309+ print "$reldir ... \n" if $verbosity;
1310+
1311+ my @subdirs;
1312+ while (defined(my $fn = readdir(DP))) {
1313+ next if $fn =~ /^\.\.?$/ || -l $fn;
1314+ if (-d _) {
1315+ push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
1316+ next;
1317+ }
1318+ next unless -f _;
1319+
1320+ my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10];
1321+ my $disk_id = $disk_id{$dev} or next;
1322+ $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!;
1323+ my($sum4, $dbsum4, $sum5, $dbsum5);
1324+ my $dbsumcnt = 0;
1325+ while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) {
1326+ if ($sum_type == 4) {
1327+ $dbsum4 = $checksum;
1328+ $dbsumcnt++;
1329+ } elsif ($sum_type == 5) {
1330+ $dbsum5 = $checksum;
1331+ $dbsumcnt++;
1332+ }
1333+ }
1334+ $sel_sum_H->finish;
1335+
1336+ next if !$check_opt && $dbsumcnt == 2;
1337+
1338+ if (!$check_opt || $dbsumcnt || $verbosity > 2) {
1339+ if (!open(IN, $fn)) {
1340+ print STDERR "Unable to read $fn: $!\n";
1341+ next;
1342+ }
1343+
1344+ while (1) {
1345+ while (sysread(IN, $_, 64*1024)) {
1346+ $md4->add($_);
1347+ $md5->add($_);
1348+ }
1349+ $sum4 = $md4->digest;
1350+ $sum5 = $md5->digest;
1351+ print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2;
1352+ print " $fn" if $verbosity > 1;
1353+ my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10];
1354+ last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
1355+ $ino = $ino2;
1356+ $size = $size2;
1357+ $mtime = $mtime2;
1358+ $ctime = $ctime2;
1359+ sysseek(IN, 0, 0);
1360+ print " REREADING\n" if $verbosity > 1;
1361+ }
1362+
1363+ close IN;
1364+ } elsif ($verbosity > 1) {
1365+ print "_$fn";
1366+ }
1367+
1368+ if ($check_opt) {
1369+ my $dif;
1370+ if ($dbsumcnt == 0) {
1371+ $dif = ' --MISSING--';
1372+ } else {
1373+ $dif = '';
1374+ if (!defined $dbsum4) {
1375+ $dif .= ' -NO-MD4-';
1376+ } elsif ($sum4 ne $dbsum4) {
1377+ $dif .= ' -MD4-CHANGED-';
1378+ }
1379+ if (!defined $dbsum5) {
1380+ $dif .= ' ---NO-MD5---';
1381+ } elsif ($sum5 ne $dbsum5) {
1382+ $dif .= ' -MD5-CHANGED-';
1383+ }
1384+ if ($dif eq '') {
1385+ print " ====OK====\n" if $verbosity > 1;
1386+ next;
1387+ }
1388+ $dif =~ s/MD4-CHANGED MD5-//;
1389+ }
1390+ if ($verbosity < 2) {
1391+ print $verbosity ? ' ' : "$reldir/";
1392+ print $fn;
1393+ }
1394+ print $dif, "\n";
1395+ $exit_code = 1;
1396+ } else {
1397+ print "\n" if $verbosity > 1;
1398+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4);
1399+ $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5);
1400+ }
1401+ }
1402+
1403+ closedir DP;
1404+
1405+ unshift(@dirs, sort @subdirs) if $recurse_opt;
1406+}
1407+
1408+exit $exit_code;
1409+
1410+sub usage
1411+{
1412+ die <<EOT;
1413+Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS]
1414+
1415+Options:
1416+ --db=FILE Specify the config FILE to read for the DB info.
bc3fcf1d
WD
1417+ --init Create (recreate) needed tables (making them empty).
1418+ No DIR scanning, but can be combined with --mounts.
1419+ -m, --mounts Update mount info. Does no DIR scanning.
cbdf862c
WD
1420+ -r, --recurse Scan files in subdirectories too.
1421+ -c, --check Check if the checksums are right (doesn't update).
1422+ -v, --verbose Mention what we're doing. Repeat for more info.
1423+ -h, --help Display this help message.
1424+EOT
1425+}