Commit | Line | Data |
---|---|---|
cbdf862c WD |
1 | Added some DB-access routines to help rsync keep extra filesystem info |
2 | about the files it is dealing with. This adds both the --db=CONFIG_FILE | |
3 | option and the "db config" daemon parameter. | |
4 | ||
5 | For the moment this only adds checksum caching when the --checksum option | |
bc3fcf1d | 6 | is used. Future improvements may include: |
cbdf862c WD |
7 | |
8 | - Updating of MD5 checksums when transferring any file, even w/o -c. | |
bc3fcf1d WD |
9 | We should be able to extend this to work for MD4 checksums too if we |
10 | make the sender force checksum_seed to 0 when using a DB and having | |
11 | the receiving side check to see if it got a 0 checksum_seed. (We | |
12 | probably don't want to compute 2 MD4 checksums for the case where | |
13 | the checksum_seed is non-zero.) | |
cbdf862c WD |
14 | |
15 | - Caching of path info that allows for the finding of files to use for | |
16 | moving/linking/copying/alternate-basis-use. | |
17 | ||
18 | - Extend DB support beyond MySQL and SQLite (PostgreSQL?). | |
19 | ||
20 | To use this patch, run these commands for a successful build: | |
21 | ||
22 | patch -p1 <patches/remote-option.diff | |
23 | patch -p1 <patches/db.diff | |
24 | ./configure (optional if already run) | |
25 | make | |
26 | ||
27 | diff --git a/Makefile.in b/Makefile.in | |
28 | --- a/Makefile.in | |
29 | +++ b/Makefile.in | |
30 | @@ -35,7 +35,7 @@ ZLIBOBJ=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \ | |
31 | OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \ | |
32 | util.o main.o checksum.o match.o syscall.o log.o backup.o | |
33 | OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \ | |
34 | - fileio.o batch.o clientname.o chmod.o acls.o xattrs.o | |
35 | + fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o | |
36 | OBJS3=progress.o pipe.o | |
37 | DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o | |
38 | popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ | |
39 | diff --git a/checksum.c b/checksum.c | |
40 | --- a/checksum.c | |
41 | +++ b/checksum.c | |
42 | @@ -23,6 +23,7 @@ | |
43 | ||
44 | extern int checksum_seed; | |
45 | extern int protocol_version; | |
46 | +extern int use_db; | |
47 | ||
48 | int csum_length = SHORT_SUM_LENGTH; /* initial value */ | |
49 | ||
50 | @@ -100,10 +101,10 @@ void get_checksum2(char *buf, int32 len, char *sum) | |
51 | } | |
52 | } | |
53 | ||
54 | -void file_checksum(char *fname, char *sum, OFF_T size) | |
55 | +void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum) | |
56 | { | |
57 | struct map_struct *buf; | |
58 | - OFF_T i, len = size; | |
59 | + OFF_T i, len = st_p->st_size; | |
60 | md_context m; | |
61 | int32 remainder; | |
62 | int fd; | |
63 | @@ -114,7 +115,7 @@ void file_checksum(char *fname, char *sum, OFF_T size) | |
64 | if (fd == -1) | |
65 | return; | |
66 | ||
67 | - buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK); | |
68 | + buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK); | |
69 | ||
70 | if (protocol_version >= 30) { | |
71 | md5_begin(&m); | |
72 | @@ -148,6 +149,9 @@ void file_checksum(char *fname, char *sum, OFF_T size) | |
73 | mdfour_result(&m, (uchar *)sum); | |
74 | } | |
75 | ||
76 | + if (use_db) | |
77 | + db_set_checksum(fname, st_p, sum); | |
78 | + | |
79 | close(fd); | |
80 | unmap_file(buf); | |
81 | } | |
82 | diff --git a/cleanup.c b/cleanup.c | |
83 | --- a/cleanup.c | |
84 | +++ b/cleanup.c | |
85 | @@ -27,6 +27,7 @@ extern int am_daemon; | |
86 | extern int io_error; | |
87 | extern int keep_partial; | |
88 | extern int got_xfer_error; | |
89 | +extern int use_db; | |
90 | extern char *partial_dir; | |
91 | extern char *logfile_name; | |
7f0bf1cb WD |
92 | |
93 | @@ -124,6 +125,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) | |
cbdf862c WD |
94 | /* FALLTHROUGH */ |
95 | #include "case_N.h" | |
96 | ||
97 | + if (use_db) | |
98 | + db_disconnect(); | |
99 | + | |
100 | + /* FALLTHROUGH */ | |
101 | +#include "case_N.h" | |
102 | + | |
103 | if (cleanup_child_pid != -1) { | |
104 | int status; | |
105 | int pid = wait_process(cleanup_child_pid, &status, WNOHANG); | |
106 | diff --git a/clientserver.c b/clientserver.c | |
107 | --- a/clientserver.c | |
108 | +++ b/clientserver.c | |
bc3fcf1d | 109 | @@ -42,13 +42,16 @@ extern int numeric_ids; |
cbdf862c WD |
110 | extern int filesfrom_fd; |
111 | extern int remote_protocol; | |
112 | extern int protocol_version; | |
113 | +extern int always_checksum; | |
114 | extern int io_timeout; | |
115 | extern int no_detach; | |
bc3fcf1d | 116 | +extern int use_db; |
cbdf862c | 117 | extern int write_batch; |
bc3fcf1d | 118 | extern int default_af_hint; |
cbdf862c WD |
119 | extern int logfile_format_has_i; |
120 | extern int logfile_format_has_o_or_i; | |
121 | extern mode_t orig_umask; | |
122 | +extern char *db_config; | |
123 | extern char *bind_address; | |
124 | extern char *sockopts; | |
125 | extern char *config_file; | |
e66d6d51 | 126 | @@ -552,6 +555,9 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) |
cbdf862c | 127 | |
bc3fcf1d WD |
128 | log_init(1); |
129 | ||
130 | + if (*lp_db_config(i)) | |
131 | + db_read_config(FLOG, lp_db_config(i)); | |
cbdf862c | 132 | + |
bc3fcf1d WD |
133 | #ifdef HAVE_PUTENV |
134 | if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) { | |
135 | char *modname, *modpath, *hostaddr, *hostname, *username; | |
e66d6d51 | 136 | @@ -768,6 +774,10 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) |
cbdf862c | 137 | |
bc3fcf1d WD |
138 | am_server = 1; /* Don't let someone try to be tricky. */ |
139 | quiet = 0; | |
140 | + db_config = NULL; | |
141 | + if (!always_checksum) | |
142 | + use_db = 0; | |
143 | + | |
144 | if (lp_ignore_errors(module_id)) | |
145 | ignore_errors = 1; | |
146 | if (write_batch < 0) | |
cbdf862c WD |
147 | diff --git a/configure.in b/configure.in |
148 | --- a/configure.in | |
149 | +++ b/configure.in | |
bc3fcf1d WD |
150 | @@ -314,7 +314,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \ |
151 | sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \ | |
152 | netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \ | |
153 | sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \ | |
154 | - popt.h popt/popt.h) | |
155 | + popt.h popt/popt.h mysql/mysql.h sqlite3.h) | |
156 | AC_HEADER_MAJOR | |
157 | ||
158 | AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[ | |
159 | @@ -969,6 +969,29 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x" | |
cbdf862c WD |
160 | fi |
161 | fi | |
162 | ||
bc3fcf1d WD |
163 | +AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0) |
164 | +if test x$MYSQL_CONFIG = x1; then | |
165 | + AC_MSG_CHECKING(for mysql version >= 4) | |
166 | + mysql_version=`mysql_config --version` | |
167 | + mysql_major_version=`echo $mysql_version | sed 's/\..*//'` | |
168 | + if test $mysql_major_version -lt 4; then | |
169 | + AC_MSG_RESULT(no.. skipping MySQL) | |
170 | + else | |
171 | + AC_MSG_RESULT(yes) | |
172 | + | |
173 | + MYSQL_CFLAGS=`mysql_config --cflags` | |
174 | + MYSQL_LIBS=`mysql_config --libs` | |
175 | + | |
176 | + CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS" | |
177 | + LIBS="$MYSQL_LIBS $LIBS" | |
178 | + | |
179 | + AC_CHECK_LIB(mysqlclient, mysql_init) | |
180 | + fi | |
181 | +fi | |
182 | + | |
183 | +AC_CHECK_LIB(sqlite3, sqlite3_open) | |
184 | +AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2) | |
cbdf862c WD |
185 | + |
186 | case "$CC" in | |
187 | ' checker'*|checker*) | |
188 | AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.]) | |
189 | diff --git a/db.c b/db.c | |
190 | new file mode 100644 | |
191 | --- /dev/null | |
192 | +++ b/db.c | |
bc3fcf1d | 193 | @@ -0,0 +1,566 @@ |
cbdf862c WD |
194 | +/* |
195 | + * Routines to access extended file info via DB. | |
196 | + * | |
197 | + * Copyright (C) 2008 Wayne Davison | |
198 | + * | |
199 | + * This program is free software; you can redistribute it and/or modify | |
200 | + * it under the terms of the GNU General Public License as published by | |
201 | + * the Free Software Foundation; either version 3 of the License, or | |
202 | + * (at your option) any later version. | |
203 | + * | |
204 | + * This program is distributed in the hope that it will be useful, | |
205 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
206 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
207 | + * GNU General Public License for more details. | |
208 | + * | |
209 | + * You should have received a copy of the GNU General Public License along | |
210 | + * with this program; if not, visit the http://fsf.org website. | |
211 | + */ | |
212 | + | |
213 | +#include "rsync.h" | |
214 | +#include "ifuncs.h" | |
215 | + | |
bc3fcf1d | 216 | +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT |
cbdf862c | 217 | +#define USE_MYSQL |
cbdf862c WD |
218 | +#include <mysql/mysql.h> |
219 | +#include <mysql/errmsg.h> | |
220 | +#endif | |
bc3fcf1d WD |
221 | + |
222 | +#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3 | |
223 | +#define USE_SQLITE | |
cbdf862c | 224 | +#include <sqlite3.h> |
bc3fcf1d WD |
225 | +#ifndef HAVE_SQLITE3_OPEN_V2 |
226 | +#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \ | |
227 | + sqlite3_open(dbname, dbhptr) | |
228 | +#endif | |
229 | +#ifndef HAVE_SQLITE3_PREPARE_V2 | |
230 | +#define sqlite3_prepare_v2 sqlite3_prepare | |
231 | +#endif | |
cbdf862c WD |
232 | +#endif |
233 | + | |
234 | +extern int protocol_version; | |
235 | +extern int checksum_len; | |
236 | + | |
237 | +#define DB_TYPE_NONE 0 | |
238 | +#define DB_TYPE_MYSQL 1 | |
239 | +#define DB_TYPE_SQLITE 2 | |
240 | + | |
241 | +int use_db = DB_TYPE_NONE; | |
242 | + | |
243 | +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL; | |
244 | +static unsigned int dbport = 0; | |
245 | + | |
246 | +static union { | |
247 | +#ifdef USE_MYSQL | |
248 | + MYSQL *mysql; | |
249 | +#endif | |
250 | +#ifdef USE_SQLITE | |
251 | + sqlite3 *sqlite; | |
252 | +#endif | |
253 | + void *all; | |
254 | +} dbh; | |
255 | + | |
256 | +#define SEL_DEV 0 | |
257 | +#define SEL_SUM 1 | |
258 | +#define REP_SUM 2 | |
259 | +#define MAX_PREP_CNT 3 | |
260 | + | |
261 | +static union { | |
262 | +#ifdef USE_MYSQL | |
263 | + MYSQL_STMT *mysql; | |
264 | +#endif | |
265 | +#ifdef USE_SQLITE | |
266 | + sqlite3_stmt *sqlite; | |
267 | +#endif | |
268 | + void *all; | |
269 | +} statements[MAX_PREP_CNT]; | |
270 | + | |
271 | +static int md_num; | |
272 | +static enum logcode log_code; | |
273 | + | |
bc3fcf1d | 274 | +#ifdef USE_MYSQL |
cbdf862c WD |
275 | +static unsigned int bind_disk_id; |
276 | +static unsigned long long bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime; | |
bc3fcf1d WD |
277 | +static char bind_sum[MAX_DIGEST_LEN]; |
278 | +#endif | |
279 | +static char bind_thishost[256]; | |
cbdf862c WD |
280 | +static int bind_thishost_len; |
281 | + | |
282 | +static unsigned int prior_disk_id = 0; | |
283 | +static unsigned long long prior_devno = 0; | |
284 | + | |
285 | +int db_read_config(enum logcode code, const char *config_file) | |
286 | +{ | |
287 | + char buf[2048], *cp; | |
288 | + FILE *fp; | |
289 | + int lineno = 0; | |
290 | + | |
291 | + log_code = code; | |
292 | + | |
293 | + bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost); | |
294 | + | |
295 | + if (!(fp = fopen(config_file, "r"))) { | |
296 | + rsyserr(log_code, errno, "unable to open %s", config_file); | |
297 | + return 0; | |
298 | + } | |
299 | + while (fgets(buf, sizeof buf, fp)) { | |
300 | + lineno++; | |
301 | + if ((cp = strchr(buf, '#')) == NULL | |
302 | + && (cp = strchr(buf, '\r')) == NULL | |
303 | + && (cp = strchr(buf, '\n')) == NULL) | |
304 | + cp = buf + strlen(buf); | |
305 | + while (cp != buf && isSpace(cp-1)) cp--; | |
306 | + *cp = '\0'; | |
307 | + | |
308 | + if (!*buf) | |
309 | + continue; | |
310 | + | |
311 | + if (!(cp = strchr(buf, ':'))) | |
312 | + goto invalid_line; | |
313 | + *cp++ = '\0'; | |
314 | + | |
315 | + while (isSpace(cp)) cp++; | |
316 | + if (strcasecmp(buf, "dbhost") == 0) | |
317 | + dbhost = strdup(cp); | |
318 | + else if (strcasecmp(buf, "dbuser") == 0) | |
319 | + dbuser = strdup(cp); | |
320 | + else if (strcasecmp(buf, "dbpass") == 0) | |
321 | + dbpass = strdup(cp); | |
322 | + else if (strcasecmp(buf, "dbname") == 0) | |
323 | + dbname = strdup(cp); | |
324 | + else if (strcasecmp(buf, "dbport") == 0) | |
325 | + dbport = atoi(cp); | |
326 | + else if (strcasecmp(buf, "thishost") == 0) | |
327 | + bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost); | |
328 | + else if (strcasecmp(buf, "dbtype") == 0) { | |
329 | +#ifdef USE_MYSQL | |
330 | + if (strcasecmp(cp, "mysql") == 0) { | |
331 | + use_db = DB_TYPE_MYSQL; | |
332 | + continue; | |
333 | + } | |
334 | +#endif | |
335 | +#ifdef USE_SQLITE | |
336 | + if (strcasecmp(cp, "sqlite") == 0) { | |
337 | + use_db = DB_TYPE_SQLITE; | |
338 | + continue; | |
339 | + } | |
340 | +#endif | |
341 | + rprintf(log_code, | |
342 | + "Unsupported dbtype on line #%d in %s.\n", | |
343 | + lineno, config_file); | |
344 | + use_db = DB_TYPE_NONE; | |
345 | + return 0; | |
346 | + } else { | |
347 | + invalid_line: | |
348 | + rprintf(log_code, "Invalid line #%d in %s\n", | |
349 | + lineno, config_file); | |
350 | + use_db = DB_TYPE_NONE; | |
351 | + return 0; | |
352 | + } | |
353 | + } | |
354 | + fclose(fp); | |
355 | + | |
356 | + if (bind_thishost_len >= (int)sizeof bind_thishost) | |
357 | + bind_thishost_len = sizeof bind_thishost - 1; | |
358 | + | |
359 | + if (!use_db || !dbname) { | |
360 | + rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file); | |
361 | + use_db = DB_TYPE_NONE; | |
362 | + return 0; | |
363 | + } | |
364 | + | |
365 | + md_num = protocol_version >= 30 ? 5 : 4; | |
366 | + | |
367 | + return 1; | |
368 | +} | |
369 | + | |
370 | +#ifdef USE_MYSQL | |
371 | +static MYSQL_STMT *prepare_mysql(MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...) | |
372 | +{ | |
373 | + va_list ap; | |
374 | + char *query; | |
375 | + int qlen, param_cnt; | |
376 | + MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql); | |
377 | + | |
378 | + if (stmt == NULL) | |
379 | + out_of_memory("prepare_mysql"); | |
380 | + | |
381 | + va_start(ap, fmt); | |
382 | + qlen = vasprintf(&query, fmt, ap); | |
383 | + va_end(ap); | |
384 | + if (qlen < 0) | |
385 | + out_of_memory("prepare_mysql"); | |
386 | + | |
387 | + if (mysql_stmt_prepare(stmt, query, qlen) != 0) { | |
388 | + rprintf(log_code, "Prepare failed: %s\n", mysql_stmt_error(stmt)); | |
389 | + return NULL; | |
390 | + } | |
391 | + free(query); | |
392 | + | |
393 | + if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) { | |
394 | + rprintf(log_code, "Parameters in statement = %d, bind vars = %d\n", | |
395 | + param_cnt, bind_cnt); | |
396 | + return NULL; | |
397 | + } | |
398 | + if (bind_cnt) | |
399 | + mysql_stmt_bind_param(stmt, binds); | |
400 | + | |
401 | + return stmt; | |
402 | +} | |
403 | +#endif | |
404 | + | |
405 | +#ifdef USE_MYSQL | |
406 | +static int db_connect_mysql(void) | |
407 | +{ | |
408 | + MYSQL_BIND binds[10]; | |
409 | + | |
410 | + if (!(dbh.mysql = mysql_init(NULL))) | |
411 | + out_of_memory("db_read_config"); | |
412 | + | |
413 | + if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, dbname, dbport, NULL, 0)) | |
414 | + return 0; | |
415 | + | |
416 | + memset(binds, 0, sizeof binds); | |
417 | + binds[0].buffer_type = MYSQL_TYPE_LONGLONG; | |
418 | + binds[0].buffer = &bind_devno; | |
419 | + binds[1].buffer_type = MYSQL_TYPE_STRING; | |
420 | + binds[1].buffer = &bind_thishost; | |
421 | + binds[1].buffer_length = bind_thishost_len; | |
422 | + statements[SEL_DEV].mysql = prepare_mysql(binds, 2, | |
423 | + "SELECT disk_id" | |
424 | + " FROM disk" | |
425 | + " WHERE devno = ? AND host = ? AND mounted = 1"); | |
426 | + if (!statements[SEL_DEV].mysql) | |
427 | + return 0; | |
428 | + | |
429 | + memset(binds, 0, sizeof binds); | |
430 | + binds[0].buffer_type = MYSQL_TYPE_LONG; | |
431 | + binds[0].buffer = &bind_disk_id; | |
432 | + binds[1].buffer_type = MYSQL_TYPE_LONGLONG; | |
433 | + binds[1].buffer = &bind_ino; | |
434 | + binds[2].buffer_type = MYSQL_TYPE_LONGLONG; | |
435 | + binds[2].buffer = &bind_size; | |
436 | + binds[3].buffer_type = MYSQL_TYPE_LONGLONG; | |
437 | + binds[3].buffer = &bind_mtime; | |
438 | + binds[4].buffer_type = MYSQL_TYPE_LONGLONG; | |
439 | + binds[4].buffer = &bind_ctime; | |
440 | + statements[SEL_SUM].mysql = prepare_mysql(binds, 5, | |
441 | + "SELECT checksum" | |
442 | + " FROM inode_map" | |
443 | + " WHERE disk_id = ? AND ino = ? AND sum_type = %d" | |
444 | + " AND size = ? AND mtime = ? AND ctime = ?", | |
445 | + md_num); | |
446 | + if (!statements[SEL_SUM].mysql) | |
447 | + return 0; | |
448 | + | |
449 | + memset(binds, 0, sizeof binds); | |
450 | + binds[0].buffer_type = MYSQL_TYPE_LONG; | |
451 | + binds[0].buffer = &bind_disk_id; | |
452 | + binds[1].buffer_type = MYSQL_TYPE_LONGLONG; | |
453 | + binds[1].buffer = &bind_ino; | |
454 | + binds[2].buffer_type = binds[6].buffer_type = MYSQL_TYPE_LONGLONG; | |
455 | + binds[2].buffer = binds[6].buffer = &bind_size; | |
456 | + binds[3].buffer_type = binds[7].buffer_type = MYSQL_TYPE_LONGLONG; | |
457 | + binds[3].buffer = binds[7].buffer = &bind_mtime; | |
458 | + binds[4].buffer_type = binds[8].buffer_type = MYSQL_TYPE_LONGLONG; | |
459 | + binds[4].buffer = binds[8].buffer = &bind_ctime; | |
460 | + binds[5].buffer_type = binds[9].buffer_type = MYSQL_TYPE_BLOB; | |
461 | + binds[5].buffer = binds[9].buffer = &bind_sum; | |
462 | + binds[5].buffer_length = binds[9].buffer_length = checksum_len; | |
463 | + statements[REP_SUM].mysql = prepare_mysql(binds, 10, | |
464 | + "INSERT INTO inode_map" | |
465 | + " SET disk_id = ?, ino = ?, sum_type = %d," | |
466 | + " size = ?, mtime = ?, ctime = ?, checksum = ?" | |
467 | + " ON DUPLICATE KEY" | |
468 | + " UPDATE size = ?, mtime = ?, ctime = ?, checksum = ?", | |
469 | + md_num, md_num); | |
470 | + if (!statements[REP_SUM].mysql) | |
471 | + return 0; | |
472 | + | |
473 | + return 1; | |
474 | +} | |
475 | +#endif | |
476 | + | |
477 | +#ifdef USE_SQLITE | |
478 | +static int db_connect_sqlite(void) | |
479 | +{ | |
480 | + char *sql; | |
481 | + | |
cbdf862c WD |
482 | + if (sqlite3_open_v2(dbname, &dbh.sqlite, SQLITE_OPEN_READWRITE, NULL) != 0) |
483 | + return 0; | |
cbdf862c WD |
484 | + |
485 | + sql = "SELECT disk_id" | |
486 | + " FROM disk" | |
487 | + " WHERE devno = ? AND host = ? AND mounted = 1"; | |
488 | + if (sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_DEV].sqlite, NULL) != 0) | |
489 | + return 0; | |
490 | + | |
491 | + if (asprintf(&sql, | |
492 | + "SELECT checksum" | |
493 | + " FROM inode_map" | |
494 | + " WHERE disk_id = ? AND ino = ? AND sum_type = %d" | |
495 | + " AND size = ? AND mtime = ? AND ctime = ?", | |
496 | + md_num) < 0 | |
497 | + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[SEL_SUM].sqlite, NULL) != 0) | |
498 | + return 0; | |
499 | + free(sql); | |
500 | + | |
501 | + if (asprintf(&sql, | |
502 | + "INSERT OR REPLACE INTO inode_map" | |
503 | + " (disk_id, ino, sum_type, size, mtime, ctime, checksum)" | |
504 | + " VALUES(?, ?, %d, ?, ?, ?, ?)", | |
505 | + md_num) < 0 | |
506 | + || sqlite3_prepare_v2(dbh.sqlite, sql, -1, &statements[REP_SUM].sqlite, NULL) != 0) | |
507 | + return 0; | |
508 | + free(sql); | |
509 | + | |
510 | + return 1; | |
511 | +} | |
512 | +#endif | |
513 | + | |
514 | +int db_connect(void) | |
515 | +{ | |
516 | + switch (use_db) { | |
517 | +#ifdef USE_MYSQL | |
518 | + case DB_TYPE_MYSQL: | |
519 | + if (db_connect_mysql()) | |
520 | + return 1; | |
521 | + break; | |
522 | +#endif | |
523 | +#ifdef USE_SQLITE | |
524 | + case DB_TYPE_SQLITE: | |
525 | + if (db_connect_sqlite()) | |
526 | + return 1; | |
527 | + break; | |
528 | +#endif | |
529 | + } | |
530 | + | |
531 | + rprintf(log_code, "Unable to connect to DB\n"); | |
532 | + db_disconnect(); | |
533 | + use_db = DB_TYPE_NONE; | |
534 | + | |
535 | + return 0; | |
536 | +} | |
537 | + | |
538 | +void db_disconnect(void) | |
539 | +{ | |
540 | + int ndx; | |
541 | + | |
542 | + if (!dbh.all) | |
543 | + return; | |
544 | + | |
545 | + for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) { | |
546 | + if (statements[ndx].all) { | |
547 | + switch (use_db) { | |
548 | +#ifdef USE_MYSQL | |
549 | + case DB_TYPE_MYSQL: | |
550 | + mysql_stmt_close(statements[ndx].mysql); | |
551 | + break; | |
552 | +#endif | |
553 | +#ifdef USE_SQLITE | |
554 | + case DB_TYPE_SQLITE: | |
555 | + sqlite3_finalize(statements[ndx].sqlite); | |
556 | + break; | |
557 | +#endif | |
558 | + } | |
559 | + statements[ndx].all = NULL; | |
560 | + } | |
561 | + } | |
562 | + | |
563 | + switch (use_db) { | |
564 | +#ifdef USE_MYSQL | |
565 | + case DB_TYPE_MYSQL: | |
566 | + mysql_close(dbh.mysql); | |
567 | + break; | |
568 | +#endif | |
569 | +#ifdef USE_SQLITE | |
570 | + case DB_TYPE_SQLITE: | |
571 | + sqlite3_close(dbh.sqlite); | |
572 | + break; | |
573 | +#endif | |
574 | + } | |
575 | + | |
576 | + dbh.all = NULL; | |
577 | +} | |
578 | + | |
bc3fcf1d | 579 | +#ifdef USE_MYSQL |
cbdf862c WD |
580 | +static MYSQL_STMT *exec_mysql(int ndx) |
581 | +{ | |
582 | + MYSQL_STMT *stmt = statements[ndx].mysql; | |
583 | + int rc; | |
584 | + | |
585 | + if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) { | |
586 | + db_disconnect(); | |
587 | + if (db_connect()) { | |
588 | + stmt = statements[ndx].mysql; | |
589 | + rc = mysql_stmt_execute(stmt); | |
590 | + } | |
591 | + } | |
592 | + if (rc != 0) { | |
593 | + rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt)); | |
594 | + return NULL; | |
595 | + } | |
596 | + | |
597 | + return stmt; | |
598 | +} | |
bc3fcf1d | 599 | +#endif |
cbdf862c | 600 | + |
bc3fcf1d | 601 | +#ifdef USE_MYSQL |
cbdf862c WD |
602 | +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx) |
603 | +{ | |
604 | + unsigned long length[32]; | |
605 | + my_bool is_null[32], error[32]; | |
606 | + MYSQL_STMT *stmt; | |
607 | + int i, rc; | |
608 | + | |
609 | + if (bind_cnt > 32) | |
610 | + exit_cleanup(RERR_UNSUPPORTED); | |
611 | + | |
612 | + if ((stmt = exec_mysql(ndx)) == NULL) | |
613 | + return 0; | |
614 | + | |
615 | + for (i = 0; i < bind_cnt; i++) { | |
616 | + binds[i].is_null = &is_null[i]; | |
617 | + binds[i].length = &length[i]; | |
618 | + binds[i].error = &error[i]; | |
619 | + } | |
620 | + mysql_stmt_bind_result(stmt, binds); | |
621 | + | |
622 | + if ((rc = mysql_stmt_fetch(stmt)) != 0) { | |
623 | + if (rc != MYSQL_NO_DATA) { | |
624 | + rprintf(log_code, "SELECT fetch failed: %s\n", | |
625 | + mysql_stmt_error(stmt)); | |
626 | + } | |
627 | + mysql_stmt_free_result(stmt); | |
628 | + return 0; | |
629 | + } | |
630 | + | |
631 | + mysql_stmt_free_result(stmt); | |
632 | + | |
633 | + return is_null[0] ? 0 : 1; | |
634 | +} | |
bc3fcf1d | 635 | +#endif |
cbdf862c WD |
636 | + |
637 | +static void get_disk_id(unsigned long long devno) | |
638 | +{ | |
639 | + switch (use_db) { | |
640 | +#ifdef USE_MYSQL | |
641 | + case DB_TYPE_MYSQL: { | |
642 | + MYSQL_BIND binds[1]; | |
643 | + | |
644 | + bind_devno = devno; /* The one variable SEL_DEV input value. */ | |
645 | + | |
646 | + /* Bind where to put the output. */ | |
647 | + binds[0].buffer_type = MYSQL_TYPE_LONG; | |
648 | + binds[0].buffer = &prior_disk_id; | |
649 | + if (!fetch_mysql(binds, 1, SEL_DEV)) | |
650 | + prior_disk_id = 0; | |
651 | + break; | |
652 | + } | |
653 | +#endif | |
654 | +#ifdef USE_SQLITE | |
655 | + case DB_TYPE_SQLITE: { | |
656 | + sqlite3_stmt *stmt = statements[SEL_DEV].sqlite; | |
657 | + sqlite3_bind_int64(stmt, 1, devno); | |
658 | + sqlite3_bind_text(stmt, 2, bind_thishost, bind_thishost_len, SQLITE_STATIC); | |
659 | + if (sqlite3_step(stmt) == SQLITE_ROW) | |
660 | + prior_disk_id = sqlite3_column_int(stmt, 0); | |
661 | + else | |
662 | + prior_disk_id = 0; | |
663 | + sqlite3_reset(stmt); | |
664 | + break; | |
665 | + } | |
666 | +#endif | |
667 | + } | |
668 | + | |
669 | + prior_devno = devno; | |
670 | +} | |
671 | + | |
672 | +int db_get_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, char *sum) | |
673 | +{ | |
674 | + if (prior_devno != st_p->st_dev) | |
675 | + get_disk_id(st_p->st_dev); | |
676 | + if (prior_disk_id == 0) | |
677 | + return 0; | |
678 | + | |
679 | + switch (use_db) { | |
680 | +#ifdef USE_MYSQL | |
681 | + case DB_TYPE_MYSQL: { | |
682 | + MYSQL_BIND binds[1]; | |
683 | + | |
684 | + bind_disk_id = prior_disk_id; | |
685 | + bind_ino = st_p->st_ino; | |
686 | + bind_size = st_p->st_size; | |
687 | + bind_mtime = st_p->st_mtime; | |
688 | + bind_ctime = st_p->st_ctime; | |
689 | + | |
690 | + binds[0].buffer_type = MYSQL_TYPE_BLOB; | |
691 | + binds[0].buffer = sum; | |
692 | + binds[0].buffer_length = checksum_len; | |
693 | + return fetch_mysql(binds, 1, SEL_SUM); | |
694 | + } | |
695 | +#endif | |
696 | +#ifdef USE_SQLITE | |
697 | + case DB_TYPE_SQLITE: { | |
698 | + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite; | |
699 | + sqlite3_bind_int(stmt, 1, prior_disk_id); | |
700 | + sqlite3_bind_int64(stmt, 2, st_p->st_ino); | |
701 | + sqlite3_bind_int64(stmt, 3, st_p->st_size); | |
702 | + sqlite3_bind_int64(stmt, 4, st_p->st_mtime); | |
703 | + sqlite3_bind_int64(stmt, 5, st_p->st_ctime); | |
704 | + if (sqlite3_step(stmt) == SQLITE_ROW) { | |
705 | + int len = sqlite3_column_bytes(stmt, 0); | |
706 | + if (len > MAX_DIGEST_LEN) | |
707 | + len = MAX_DIGEST_LEN; | |
708 | + memcpy(sum, sqlite3_column_blob(stmt, 0), len); | |
709 | + sqlite3_reset(stmt); | |
710 | + return 1; | |
711 | + } | |
712 | + sqlite3_reset(stmt); | |
713 | + return 0; | |
714 | + } | |
715 | +#endif | |
716 | + } | |
717 | + | |
718 | + return 0; | |
719 | +} | |
720 | + | |
721 | +int db_set_checksum(UNUSED(const char *fname), const STRUCT_STAT *st_p, const char *sum) | |
722 | +{ | |
723 | + if (prior_devno != st_p->st_dev) | |
724 | + get_disk_id(st_p->st_dev); | |
725 | + if (prior_disk_id == 0) | |
726 | + return 0; | |
727 | + | |
728 | + switch (use_db) { | |
729 | +#ifdef USE_MYSQL | |
730 | + case DB_TYPE_MYSQL: { | |
731 | + bind_disk_id = prior_disk_id; | |
732 | + bind_ino = st_p->st_ino; | |
733 | + bind_size = st_p->st_size; | |
734 | + bind_mtime = st_p->st_mtime; | |
735 | + bind_ctime = st_p->st_ctime; | |
736 | + memcpy(bind_sum, sum, checksum_len); | |
737 | + | |
738 | + return exec_mysql(REP_SUM) != NULL; | |
739 | + } | |
740 | +#endif | |
741 | +#ifdef USE_SQLITE | |
742 | + case DB_TYPE_SQLITE: { | |
743 | + int rc; | |
744 | + sqlite3_stmt *stmt = statements[REP_SUM].sqlite; | |
745 | + sqlite3_bind_int(stmt, 1, prior_disk_id); | |
746 | + sqlite3_bind_int64(stmt, 2, st_p->st_ino); | |
747 | + sqlite3_bind_int64(stmt, 3, st_p->st_size); | |
748 | + sqlite3_bind_int64(stmt, 4, st_p->st_mtime); | |
749 | + sqlite3_bind_int64(stmt, 5, st_p->st_ctime); | |
750 | + sqlite3_bind_blob(stmt, 6, sum, checksum_len, SQLITE_TRANSIENT); | |
751 | + rc = sqlite3_step(stmt); | |
752 | + sqlite3_reset(stmt); | |
753 | + return rc == SQLITE_DONE; | |
754 | + } | |
755 | +#endif | |
756 | + } | |
757 | + | |
758 | + return 0; | |
759 | +} | |
760 | diff --git a/flist.c b/flist.c | |
761 | --- a/flist.c | |
762 | +++ b/flist.c | |
763 | @@ -54,6 +54,7 @@ extern int preserve_devices; | |
764 | extern int preserve_specials; | |
765 | extern int uid_ndx; | |
766 | extern int gid_ndx; | |
767 | +extern int use_db; | |
768 | extern int eol_nulls; | |
769 | extern int relative_paths; | |
770 | extern int implied_dirs; | |
7f0bf1cb | 771 | @@ -1250,14 +1251,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, |
cbdf862c WD |
772 | memcpy(bp + basename_len, linkname, linkname_len); |
773 | #endif | |
774 | ||
775 | - if (always_checksum && am_sender && S_ISREG(st.st_mode)) | |
776 | - file_checksum(thisname, tmp_sum, st.st_size); | |
777 | - | |
778 | if (am_sender) | |
779 | F_PATHNAME(file) = pathname; | |
780 | else if (!pool) | |
781 | F_DEPTH(file) = extra_len / EXTRA_LEN; | |
782 | ||
783 | + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { | |
784 | + if (!use_db || !db_get_checksum(thisname, &st, tmp_sum)) | |
785 | + file_checksum(thisname, &st, tmp_sum); | |
786 | + } | |
787 | + | |
788 | /* This code is only used by the receiver when it is building | |
789 | * a list of files for a delete pass. */ | |
790 | if (keep_dirlinks && linkname_len && flist) { | |
7f0bf1cb | 791 | @@ -1877,6 +1880,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) |
cbdf862c WD |
792 | | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0); |
793 | int implied_dot_dir = 0; | |
794 | ||
795 | + if (use_db) | |
796 | + db_connect(); | |
797 | + | |
798 | rprintf(FLOG, "building file list\n"); | |
799 | if (show_filelist_p()) | |
800 | start_filelist_progress("building file list"); | |
801 | diff --git a/generator.c b/generator.c | |
802 | --- a/generator.c | |
803 | +++ b/generator.c | |
804 | @@ -58,6 +58,7 @@ extern int update_only; | |
805 | extern int ignore_existing; | |
806 | extern int ignore_non_existing; | |
807 | extern int inplace; | |
808 | +extern int use_db; | |
809 | extern int append_mode; | |
810 | extern int make_backups; | |
811 | extern int csum_length; | |
7f0bf1cb | 812 | @@ -721,7 +722,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) |
cbdf862c WD |
813 | of the file time to determine whether to sync */ |
814 | if (always_checksum > 0 && S_ISREG(st->st_mode)) { | |
815 | char sum[MAX_DIGEST_LEN]; | |
816 | - file_checksum(fn, sum, st->st_size); | |
817 | + if (!use_db || !db_get_checksum(fn, st, sum)) | |
818 | + file_checksum(fn, st, sum); | |
819 | return memcmp(sum, F_SUM(file), checksum_len) == 0; | |
820 | } | |
821 | ||
e2e42a01 | 822 | @@ -2204,6 +2206,9 @@ void generate_files(int f_out, const char *local_name) |
cbdf862c WD |
823 | : "enabled"); |
824 | } | |
825 | ||
826 | + if (use_db && always_checksum) | |
827 | + db_connect(); | |
828 | + | |
829 | /* Since we often fill up the outgoing socket and then just sit around | |
830 | * waiting for the other 2 processes to do their thing, we don't want | |
831 | * to exit on a timeout. If the data stops flowing, the receiver will | |
832 | diff --git a/loadparm.c b/loadparm.c | |
833 | --- a/loadparm.c | |
834 | +++ b/loadparm.c | |
835 | @@ -126,6 +126,7 @@ typedef struct | |
836 | char *auth_users; | |
837 | char *charset; | |
838 | char *comment; | |
839 | + char *db_config; | |
840 | char *dont_compress; | |
841 | char *exclude; | |
842 | char *exclude_from; | |
843 | @@ -177,6 +178,7 @@ static service sDefault = | |
844 | /* auth_users; */ NULL, | |
845 | /* charset; */ NULL, | |
846 | /* comment; */ NULL, | |
847 | + /* db_config; */ NULL, | |
848 | /* dont_compress; */ DEFAULT_DONT_COMPRESS, | |
849 | /* exclude; */ NULL, | |
850 | /* exclude_from; */ NULL, | |
851 | @@ -307,6 +309,7 @@ static struct parm_struct parm_table[] = | |
852 | {"auth users", P_STRING, P_LOCAL, &sDefault.auth_users, NULL,0}, | |
853 | {"charset", P_STRING, P_LOCAL, &sDefault.charset, NULL,0}, | |
854 | {"comment", P_STRING, P_LOCAL, &sDefault.comment, NULL,0}, | |
855 | + {"db config", P_STRING, P_LOCAL, &sDefault.db_config, NULL,0}, | |
856 | {"dont compress", P_STRING, P_LOCAL, &sDefault.dont_compress, NULL,0}, | |
857 | {"exclude from", P_STRING, P_LOCAL, &sDefault.exclude_from, NULL,0}, | |
858 | {"exclude", P_STRING, P_LOCAL, &sDefault.exclude, NULL,0}, | |
859 | @@ -400,6 +403,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Globals.rsync_port) | |
860 | FN_LOCAL_STRING(lp_auth_users, auth_users) | |
861 | FN_LOCAL_STRING(lp_charset, charset) | |
862 | FN_LOCAL_STRING(lp_comment, comment) | |
863 | +FN_LOCAL_STRING(lp_db_config, db_config) | |
864 | FN_LOCAL_STRING(lp_dont_compress, dont_compress) | |
865 | FN_LOCAL_STRING(lp_exclude, exclude) | |
866 | FN_LOCAL_STRING(lp_exclude_from, exclude_from) | |
867 | diff --git a/main.c b/main.c | |
868 | --- a/main.c | |
869 | +++ b/main.c | |
870 | @@ -49,6 +49,7 @@ extern int copy_unsafe_links; | |
871 | extern int keep_dirlinks; | |
872 | extern int preserve_hard_links; | |
873 | extern int protocol_version; | |
874 | +extern int always_checksum; | |
875 | extern int file_total; | |
876 | extern int recurse; | |
877 | extern int xfer_dirs; | |
878 | @@ -73,6 +74,7 @@ extern char *partial_dir; | |
879 | extern char *dest_option; | |
880 | extern char *basis_dir[]; | |
881 | extern char *rsync_path; | |
882 | +extern char *db_config; | |
883 | extern char *shell_cmd; | |
884 | extern char *batch_name; | |
885 | extern char *password_file; | |
7f0bf1cb | 886 | @@ -1491,6 +1493,9 @@ int main(int argc,char *argv[]) |
cbdf862c WD |
887 | exit_cleanup(RERR_SYNTAX); |
888 | } | |
889 | ||
bc3fcf1d | 890 | + if (db_config && always_checksum) |
cbdf862c WD |
891 | + db_read_config(FERROR, db_config); |
892 | + | |
893 | if (am_server) { | |
894 | set_nonblocking(STDIN_FILENO); | |
895 | set_nonblocking(STDOUT_FILENO); | |
896 | diff --git a/options.c b/options.c | |
897 | --- a/options.c | |
898 | +++ b/options.c | |
899 | @@ -92,6 +92,7 @@ int use_qsort = 0; | |
900 | char *files_from = NULL; | |
901 | int filesfrom_fd = -1; | |
902 | char *filesfrom_host = NULL; | |
903 | +char *db_config = NULL; | |
904 | int eol_nulls = 0; | |
905 | int protect_args = 0; | |
906 | int human_readable = 0; | |
bc3fcf1d WD |
907 | @@ -229,6 +230,7 @@ static void print_rsync_version(enum logcode f) |
908 | char const *links = "no "; | |
909 | char const *iconv = "no "; | |
910 | char const *ipv6 = "no "; | |
911 | + char const *db = "no "; | |
912 | STRUCT_STAT *dumstat; | |
913 | ||
914 | #if SUBPROTOCOL_VERSION != 0 | |
915 | @@ -261,6 +263,11 @@ static void print_rsync_version(enum logcode f) | |
916 | #if defined HAVE_LUTIMES && defined HAVE_UTIMES | |
917 | symtimes = ""; | |
918 | #endif | |
919 | +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT | |
920 | + db = ""; | |
921 | +#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3 | |
922 | + db = ""; | |
923 | +#endif | |
924 | ||
925 | rprintf(f, "%s version %s protocol version %d%s\n", | |
926 | RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); | |
927 | @@ -274,8 +281,8 @@ static void print_rsync_version(enum logcode f) | |
928 | (int)(sizeof (int64) * 8)); | |
929 | rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", | |
930 | got_socketpair, hardlinks, links, ipv6, have_inplace); | |
931 | - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes\n", | |
932 | - have_inplace, acls, xattrs, iconv, symtimes); | |
933 | + rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sdb\n", | |
934 | + have_inplace, acls, xattrs, iconv, symtimes, db); | |
935 | ||
936 | #ifdef MAINTAINER_MODE | |
937 | rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); | |
938 | @@ -321,6 +328,7 @@ void usage(enum logcode F) | |
cbdf862c WD |
939 | rprintf(F," -q, --quiet suppress non-error messages\n"); |
940 | rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); | |
941 | rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); | |
bc3fcf1d | 942 | + rprintf(F," --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums\n"); |
cbdf862c WD |
943 | rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); |
944 | rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); | |
945 | rprintf(F," -r, --recursive recurse into directories\n"); | |
bc3fcf1d | 946 | @@ -579,6 +587,7 @@ static struct poptOption long_options[] = { |
cbdf862c WD |
947 | {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, |
948 | {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
949 | {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, | |
950 | + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 }, | |
951 | {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, | |
952 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, | |
953 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, | |
954 | diff --git a/pipe.c b/pipe.c | |
955 | --- a/pipe.c | |
956 | +++ b/pipe.c | |
bc3fcf1d | 957 | @@ -26,6 +26,9 @@ extern int am_sender; |
cbdf862c WD |
958 | extern int am_server; |
959 | extern int blocking_io; | |
960 | extern int filesfrom_fd; | |
961 | +extern int always_checksum; | |
cbdf862c WD |
962 | +extern int use_db; |
963 | +extern char *db_config; | |
964 | extern mode_t orig_umask; | |
965 | extern char *logfile_name; | |
966 | extern int remote_option_cnt; | |
bc3fcf1d | 967 | @@ -141,6 +144,9 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, |
cbdf862c WD |
968 | logfile_close(); |
969 | } | |
970 | ||
971 | + use_db = 0; | |
972 | + db_config = NULL; | |
973 | + | |
974 | if (remote_option_cnt) { | |
975 | int rc = remote_option_cnt + 1; | |
976 | const char **rv = remote_options; | |
bc3fcf1d | 977 | @@ -148,6 +154,8 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, |
cbdf862c WD |
978 | option_error(); |
979 | exit_cleanup(RERR_SYNTAX); | |
980 | } | |
bc3fcf1d | 981 | + if (db_config && always_checksum) |
cbdf862c WD |
982 | + db_read_config(FERROR, db_config); |
983 | } | |
984 | ||
985 | if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 || | |
bc3fcf1d WD |
986 | diff --git a/rsync.yo b/rsync.yo |
987 | --- a/rsync.yo | |
988 | +++ b/rsync.yo | |
989 | @@ -317,6 +317,7 @@ to the detailed description below for a complete description. verb( | |
990 | -q, --quiet suppress non-error messages | |
991 | --no-motd suppress daemon-mode MOTD (see caveat) | |
992 | -c, --checksum skip based on checksum, not mod-time & size | |
993 | + --db=CONFIG_FILE specify a CONFIG_FILE for DB checksums | |
994 | -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) | |
995 | --no-OPTION turn off an implied OPTION (e.g. --no-D) | |
996 | -r, --recursive recurse into directories | |
997 | @@ -533,6 +534,47 @@ checksum that is generated as the file is transferred, but that | |
998 | automatic after-the-transfer verification has nothing to do with this | |
999 | option's before-the-transfer "Does this file need to be updated?" check. | |
cbdf862c | 1000 | |
bc3fcf1d WD |
1001 | +dit(bf(--db=CONFIG_FILE)) This option specifies a CONFIG_FILE to read |
1002 | +that holds connection details for a database of checksum information. | |
1003 | +When combined with the bf(--checksum) (bf(-c)) option, rsync will try to | |
1004 | +use cached checksum information from the DB, and will update it if it is | |
1005 | +missing. | |
1006 | + | |
1007 | +The currently supported DB choices are MySQL and SQLite. For example, a | |
1008 | +MySQL configuration might look like this: | |
1009 | + | |
1010 | +verb( dbtype: mysql | |
1011 | + dbhost: 127.0.0.1 | |
1012 | + dbname: rsyncdb | |
1013 | + dbuser: rsyncuser | |
1014 | + dbpass: somepass | |
1015 | + port: 3306 | |
1016 | + thishost: hostname ) | |
1017 | + | |
1018 | +And a SQLite configuration might look like this: | |
1019 | + | |
1020 | +verb( dbtype: SQLite | |
1021 | + dbname: /var/cache/rsync/sum.db ) | |
1022 | + | |
1023 | +This option only affects one side of a transfer. See the | |
1024 | +bf(--remote-option) option for a way to specify the option for both | |
1025 | +sides of the transfer (with each side reading the config file from | |
1026 | +their local filesystem). For example: | |
1027 | + | |
1028 | +verb( rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ ) | |
1029 | + | |
1030 | +See the perl script "rsyncdb" in the support directory of the source code | |
1031 | +(which may also be installed in /usr/bin) for a way to create the tables, | |
1032 | +populate the mounted-disk information, check files against their checksums, | |
1033 | +and update both the MD4 and MD5 checksums for files at the same time (since | |
1034 | +an rsync copy will only update one or the other). | |
1035 | + | |
1036 | +You can use a single MySQL DB for all your hosts if you give each one | |
1037 | +their own "thishost" name and setup their device-mapping data. Or feel | |
1038 | +free to use separate databases, separate servers, etc. See the rsync | |
1039 | +daemon's "db config" parameter for how to configure a daemon to use a DB | |
1040 | +(since a client cannot control this parameter on a daemon). | |
1041 | + | |
1042 | dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick | |
1043 | way of saying you want recursion and want to preserve almost | |
1044 | everything (with -H being a notable omission). | |
1045 | diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo | |
1046 | --- a/rsyncd.conf.yo | |
1047 | +++ b/rsyncd.conf.yo | |
1048 | @@ -270,6 +270,18 @@ is daemon. This setting has no effect if the "log file" setting is a | |
1049 | non-empty string (either set in the per-modules settings, or inherited | |
1050 | from the global settings). | |
cbdf862c | 1051 | |
bc3fcf1d WD |
1052 | +dit(bf(db config)) This parameter specifies a config file to read that |
1053 | +holds connection details for a database of checksum information. | |
1054 | + | |
1055 | +The config file will be read-in prior to any chroot restrictions, but | |
1056 | +the connection occurs from inside the chroot. This means that you | |
1057 | +should use a socket connection (e.g. 127.0.0.1 rather than localhost) | |
1058 | +for a MySQL config from inside a chroot. For SQLite, the DB file must | |
1059 | +be placed inside the chroot (though it can be placed outside the | |
1060 | +transfer dir if you configured an inside-chroot path). | |
1061 | + | |
1062 | +See the bf(--db=CONFIG_FILE) option for full details. | |
1063 | + | |
e66d6d51 | 1064 | dit(bf(max verbosity)) This parameter allows you to control |
bc3fcf1d WD |
1065 | the maximum amount of verbose information that you'll allow the daemon to |
1066 | generate (since the information goes into the log file). The default is 1, | |
1067 | diff --git a/support/rsyncdb b/support/rsyncdb | |
cbdf862c WD |
1068 | new file mode 100755 |
1069 | --- /dev/null | |
bc3fcf1d WD |
1070 | +++ b/support/rsyncdb |
1071 | @@ -0,0 +1,331 @@ | |
cbdf862c WD |
1072 | +#!/usr/bin/perl -w |
1073 | +use strict; | |
1074 | + | |
1075 | +use DBI; | |
1076 | +use Getopt::Long; | |
1077 | +use Cwd qw(abs_path cwd); | |
1078 | +use Digest::MD4; | |
1079 | +use Digest::MD5; | |
1080 | + | |
1081 | +my $MOUNT_FILE = '/etc/mtab'; | |
1082 | + | |
1083 | +&Getopt::Long::Configure('bundling'); | |
1084 | +&usage if !&GetOptions( | |
1085 | + 'db=s' => \( my $db_config ), | |
bc3fcf1d | 1086 | + 'init' => \( my $init_db ), |
cbdf862c WD |
1087 | + 'mounts|m' => \( my $update_mounts ), |
1088 | + 'recurse|r' => \( my $recurse_opt ), | |
1089 | + 'check|c' => \( my $check_opt ), | |
1090 | + 'verbose|v+' => \( my $verbosity = 0 ), | |
1091 | + 'help|h' => \( my $help_opt ), | |
1092 | +); | |
1093 | +&usage if $help_opt || !defined $db_config; | |
1094 | + | |
1095 | +my %config; | |
1096 | +open(IN, '<', $db_config) or die "Unable to open $db_config: $!\n"; | |
1097 | +while (<IN>) { | |
1098 | + s/[#\r\n].*//s; | |
1099 | + next if /^$/; | |
1100 | + my($key, $val) = /^(\S+):\s*(.*)/ or die "Unable to parse line $. of $db_config\n"; | |
1101 | + $config{$key} = $val; | |
1102 | +} | |
1103 | +close IN; | |
1104 | + | |
1105 | +die "You must define at least dbtype and dbname in $db_config\n" | |
1106 | + unless defined $config{'dbtype'} && defined $config{'dbname'}; | |
1107 | + | |
bc3fcf1d WD |
1108 | +my $sqlite = $config{'dbtype'} =~ /^sqlite$/i; |
1109 | + | |
cbdf862c WD |
1110 | +my $thishost = $config{'thishost'} || 'localhost'; |
1111 | + | |
bc3fcf1d WD |
1112 | +my $connect = 'DBI:' . $config{'dbtype'} . ':'; |
1113 | +$connect .= 'dbname=' . $config{'dbname'} if $sqlite; | |
1114 | +$connect .= 'database=' . $config{'dbname'} if !$sqlite && !$init_db; | |
cbdf862c WD |
1115 | +$connect .= ';host=' . $config{'dbhost'} if defined $config{'dbhost'}; |
1116 | +$connect .= ';port=' . $config{'dbport'} if defined $config{'dbport'}; | |
1117 | + | |
1118 | +my $dbh = DBI->connect($connect, $config{'dbuser'}, $config{'dbpass'}) | |
1119 | + or die "DB connection failed\n"; | |
1120 | + | |
1121 | +END { | |
1122 | + $dbh->disconnect if defined $dbh; | |
1123 | +} | |
1124 | + | |
bc3fcf1d WD |
1125 | +if ($init_db) { |
1126 | + my $unsigned = $sqlite ? '' : 'unsigned'; | |
1127 | + my $auto_increment = $sqlite ? 'AUTOINCREMENT' : 'AUTO_INCREMENT'; | |
1128 | + my $dbname = $config{'dbname'}; | |
1129 | + | |
1130 | + if (!$sqlite) { | |
1131 | + $dbh->do("CREATE DATABASE IF NOT EXISTS `$dbname`"); | |
1132 | + $dbh->do("USE `$dbname`"); | |
1133 | + } | |
1134 | + | |
1135 | + print "Dropping old tables (if they exist) ...\n" if $verbosity; | |
1136 | + $dbh->do("DROP TABLE IF EXISTS disk") or die $dbh->errstr; | |
1137 | + $dbh->do("DROP TABLE IF EXISTS inode_map") or die $dbh->errstr; | |
1138 | + | |
1139 | + print "Creating empty tables ...\n" if $verbosity; | |
1140 | + $dbh->do(" | |
1141 | + CREATE TABLE disk ( | |
1142 | + disk_id integer $unsigned NOT NULL PRIMARY KEY $auto_increment, | |
1143 | + devno bigint $unsigned NOT NULL, | |
1144 | + host varchar(256) NOT NULL default 'localhost', | |
1145 | + mounted tinyint NOT NULL default '1', | |
1146 | + comment varchar(256) default NULL | |
1147 | + )") or die $dbh->errstr; | |
1148 | + | |
1149 | + $dbh->do(" | |
1150 | + CREATE TABLE inode_map ( | |
1151 | + disk_id integer $unsigned NOT NULL, | |
1152 | + ino bigint $unsigned NOT NULL, | |
1153 | + size bigint $unsigned NOT NULL, | |
1154 | + mtime bigint NOT NULL, | |
1155 | + ctime bigint NOT NULL, | |
1156 | + sum_type tinyint NOT NULL default '0', | |
1157 | + checksum binary(16) NOT NULL, | |
1158 | + PRIMARY KEY (disk_id,ino,sum_type) | |
1159 | + )") or die $dbh->errstr; | |
1160 | + | |
1161 | + exit unless $update_mounts; | |
1162 | +} | |
1163 | + | |
cbdf862c WD |
1164 | +my $sel_disk_H = $dbh->prepare(" |
1165 | + SELECT disk_id, devno, mounted, comment | |
1166 | + FROM disk | |
1167 | + WHERE host = ? | |
1168 | + ") or die $dbh->errstr; | |
1169 | + | |
1170 | +my $ins_disk_H = $dbh->prepare(" | |
1171 | + INSERT INTO disk | |
1172 | + (devno, host, mounted, comment) | |
1173 | + VALUES(?, ?, ?, ?) | |
1174 | + ") or die $dbh->errstr; | |
1175 | + | |
1176 | +my $up_disk_H = $dbh->prepare(" | |
1177 | + UPDATE disk | |
1178 | + SET mounted = ? | |
1179 | + WHERE disk_id = ? | |
1180 | + ") or die $dbh->errstr; | |
1181 | + | |
bc3fcf1d | 1182 | +my $row_id = $sqlite ? 'ROWID' : 'ID'; |
cbdf862c WD |
1183 | +my $sel_lastid_H = $dbh->prepare(" |
1184 | + SELECT LAST_INSERT_$row_id() | |
1185 | + ") or die $dbh->errstr; | |
1186 | + | |
1187 | +my $sel_sum_H = $dbh->prepare(" | |
1188 | + SELECT sum_type, checksum | |
1189 | + FROM inode_map | |
1190 | + WHERE disk_id = ? AND ino = ? AND size = ? AND mtime = ? AND ctime = ? | |
1191 | + ") or die $dbh->errstr; | |
1192 | + | |
1193 | +my $rep_sum_H = $dbh->prepare(" | |
1194 | + REPLACE INTO inode_map | |
1195 | + (disk_id, ino, size, mtime, ctime, sum_type, checksum) | |
1196 | + VALUES(?, ?, ?, ?, ?, ?, ?) | |
1197 | + ") or die $dbh->errstr; | |
1198 | + | |
1199 | +my %mounts; | |
1200 | +if ($update_mounts) { | |
1201 | + open(IN, $MOUNT_FILE) or die "Unable to open $MOUNT_FILE: $!\n"; | |
1202 | + while (<IN>) { | |
1203 | + my($devname, $mnt) = (split)[0,1]; | |
1204 | + next unless $devname =~ m#^/dev#; | |
1205 | + my($devno) = (stat($mnt))[0]; | |
1206 | + if (!defined $devno) { | |
1207 | + warn "Unable to stat $mnt: $!\n"; | |
1208 | + next; | |
1209 | + } | |
1210 | + $mounts{$devno} = "$devname on $mnt"; | |
1211 | + } | |
1212 | + close IN; | |
1213 | +} | |
1214 | + | |
1215 | +my %disk_id; | |
1216 | +$sel_disk_H->execute($thishost); | |
1217 | +while (my($disk_id, $devno, $mounted, $comment) = $sel_disk_H->fetchrow_array) { | |
1218 | + if ($update_mounts) { | |
1219 | + if (defined $mounts{$devno}) { | |
1220 | + if ($comment ne $mounts{$devno}) { | |
1221 | + if ($mounted) { | |
bc3fcf1d | 1222 | + print "Umounting $comment ($thishost:$devno)\n" if $verbosity; |
cbdf862c WD |
1223 | + $up_disk_H->execute(0, $disk_id); |
1224 | + } | |
1225 | + next; | |
1226 | + } | |
1227 | + if (!$mounted) { | |
bc3fcf1d | 1228 | + print "Mounting $comment ($thishost:$devno)\n" if $verbosity; |
cbdf862c WD |
1229 | + $up_disk_H->execute(1, $disk_id); |
1230 | + } | |
1231 | + } else { | |
1232 | + if ($mounted) { | |
bc3fcf1d | 1233 | + print "Umounting $comment ($thishost:$devno)\n" if $verbosity; |
cbdf862c WD |
1234 | + $up_disk_H->execute(0, $disk_id); |
1235 | + } | |
1236 | + next; | |
1237 | + } | |
1238 | + } else { | |
1239 | + next unless $mounted; | |
1240 | + } | |
1241 | + $disk_id{$devno} = $disk_id; | |
1242 | +} | |
1243 | +$sel_disk_H->finish; | |
1244 | + | |
1245 | +if ($update_mounts) { | |
1246 | + while (my($devno, $comment) = each %mounts) { | |
1247 | + next if $disk_id{$devno}; | |
bc3fcf1d | 1248 | + print "Adding $comment ($thishost:$devno)\n" if $verbosity; |
cbdf862c WD |
1249 | + $ins_disk_H->execute($devno, $thishost, 1, $comment); |
1250 | + $sel_lastid_H->execute; | |
1251 | + ($disk_id{$devno}) = $sel_lastid_H->fetchrow_array; | |
1252 | + $sel_lastid_H->finish; | |
1253 | + } | |
bc3fcf1d | 1254 | + exit; |
cbdf862c WD |
1255 | +} |
1256 | + | |
1257 | +my $start_dir = cwd(); | |
1258 | + | |
1259 | +my @dirs = @ARGV; | |
1260 | +@dirs = '.' unless @dirs; | |
1261 | +foreach (@dirs) { | |
1262 | + $_ = abs_path($_); | |
1263 | +} | |
1264 | + | |
1265 | +$| = 1; | |
1266 | + | |
1267 | +my $exit_code = 0; | |
1268 | + | |
1269 | +my $md4 = Digest::MD4->new; | |
1270 | +my $md5 = Digest::MD5->new; | |
1271 | + | |
1272 | +while (@dirs) { | |
1273 | + my $dir = shift @dirs; | |
1274 | + | |
1275 | + if (!chdir($dir)) { | |
1276 | + warn "Unable to chdir to $dir: $!\n"; | |
1277 | + next; | |
1278 | + } | |
1279 | + if (!opendir(DP, '.')) { | |
1280 | + warn "Unable to opendir $dir: $!\n"; | |
1281 | + next; | |
1282 | + } | |
1283 | + | |
1284 | + my $reldir = $dir; | |
1285 | + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; | |
1286 | + print "$reldir ... \n" if $verbosity; | |
1287 | + | |
1288 | + my @subdirs; | |
1289 | + while (defined(my $fn = readdir(DP))) { | |
1290 | + next if $fn =~ /^\.\.?$/ || -l $fn; | |
1291 | + if (-d _) { | |
1292 | + push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/; | |
1293 | + next; | |
1294 | + } | |
1295 | + next unless -f _; | |
1296 | + | |
1297 | + my($dev,$ino,$size,$mtime,$ctime) = (stat(_))[0,1,7,9,10]; | |
1298 | + my $disk_id = $disk_id{$dev} or next; | |
1299 | + $sel_sum_H->execute($disk_id,$ino,$size,$mtime,$ctime) or die $!; | |
1300 | + my($sum4, $dbsum4, $sum5, $dbsum5); | |
1301 | + my $dbsumcnt = 0; | |
1302 | + while (my($sum_type, $checksum) = $sel_sum_H->fetchrow_array) { | |
1303 | + if ($sum_type == 4) { | |
1304 | + $dbsum4 = $checksum; | |
1305 | + $dbsumcnt++; | |
1306 | + } elsif ($sum_type == 5) { | |
1307 | + $dbsum5 = $checksum; | |
1308 | + $dbsumcnt++; | |
1309 | + } | |
1310 | + } | |
1311 | + $sel_sum_H->finish; | |
1312 | + | |
1313 | + next if !$check_opt && $dbsumcnt == 2; | |
1314 | + | |
1315 | + if (!$check_opt || $dbsumcnt || $verbosity > 2) { | |
1316 | + if (!open(IN, $fn)) { | |
1317 | + print STDERR "Unable to read $fn: $!\n"; | |
1318 | + next; | |
1319 | + } | |
1320 | + | |
1321 | + while (1) { | |
1322 | + while (sysread(IN, $_, 64*1024)) { | |
1323 | + $md4->add($_); | |
1324 | + $md5->add($_); | |
1325 | + } | |
1326 | + $sum4 = $md4->digest; | |
1327 | + $sum5 = $md5->digest; | |
1328 | + print ' ', unpack('H*', $sum4), ' ', unpack('H*', $sum5) if $verbosity > 2; | |
1329 | + print " $fn" if $verbosity > 1; | |
1330 | + my($ino2,$size2,$mtime2,$ctime2) = (stat(IN))[1,7,9,10]; | |
1331 | + last if $ino == $ino2 && $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2; | |
1332 | + $ino = $ino2; | |
1333 | + $size = $size2; | |
1334 | + $mtime = $mtime2; | |
1335 | + $ctime = $ctime2; | |
1336 | + sysseek(IN, 0, 0); | |
1337 | + print " REREADING\n" if $verbosity > 1; | |
1338 | + } | |
1339 | + | |
1340 | + close IN; | |
1341 | + } elsif ($verbosity > 1) { | |
1342 | + print "_$fn"; | |
1343 | + } | |
1344 | + | |
1345 | + if ($check_opt) { | |
1346 | + my $dif; | |
1347 | + if ($dbsumcnt == 0) { | |
1348 | + $dif = ' --MISSING--'; | |
1349 | + } else { | |
1350 | + $dif = ''; | |
1351 | + if (!defined $dbsum4) { | |
1352 | + $dif .= ' -NO-MD4-'; | |
1353 | + } elsif ($sum4 ne $dbsum4) { | |
1354 | + $dif .= ' -MD4-CHANGED-'; | |
1355 | + } | |
1356 | + if (!defined $dbsum5) { | |
1357 | + $dif .= ' ---NO-MD5---'; | |
1358 | + } elsif ($sum5 ne $dbsum5) { | |
1359 | + $dif .= ' -MD5-CHANGED-'; | |
1360 | + } | |
1361 | + if ($dif eq '') { | |
1362 | + print " ====OK====\n" if $verbosity > 1; | |
1363 | + next; | |
1364 | + } | |
1365 | + $dif =~ s/MD4-CHANGED MD5-//; | |
1366 | + } | |
1367 | + if ($verbosity < 2) { | |
1368 | + print $verbosity ? ' ' : "$reldir/"; | |
1369 | + print $fn; | |
1370 | + } | |
1371 | + print $dif, "\n"; | |
1372 | + $exit_code = 1; | |
1373 | + } else { | |
1374 | + print "\n" if $verbosity > 1; | |
1375 | + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 4, $sum4); | |
1376 | + $rep_sum_H->execute($disk_id, $ino, $size, $mtime, $ctime, 5, $sum5); | |
1377 | + } | |
1378 | + } | |
1379 | + | |
1380 | + closedir DP; | |
1381 | + | |
1382 | + unshift(@dirs, sort @subdirs) if $recurse_opt; | |
1383 | +} | |
1384 | + | |
1385 | +exit $exit_code; | |
1386 | + | |
1387 | +sub usage | |
1388 | +{ | |
1389 | + die <<EOT; | |
1390 | +Usage: rsyncsums --db=CONFIG_FILE [OPTIONS] [DIRS] | |
1391 | + | |
1392 | +Options: | |
1393 | + --db=FILE Specify the config FILE to read for the DB info. | |
bc3fcf1d WD |
1394 | + --init Create (recreate) needed tables (making them empty). |
1395 | + No DIR scanning, but can be combined with --mounts. | |
1396 | + -m, --mounts Update mount info. Does no DIR scanning. | |
cbdf862c WD |
1397 | + -r, --recurse Scan files in subdirectories too. |
1398 | + -c, --check Check if the checksums are right (doesn't update). | |
1399 | + -v, --verbose Mention what we're doing. Repeat for more info. | |
1400 | + -h, --help Display this help message. | |
1401 | +EOT | |
1402 | +} |