This patch adds the --link-by-hash=DIR option, which hard links received
files in a link farm arranged by MD4 file hash. The result is that the system
will only store one copy of the unique contents of each file, regardless of
the file's name.
This patch adds the --link-by-hash=DIR option, which hard links received
files in a link farm arranged by MD4 file hash. The result is that the system
will only store one copy of the unique contents of each file, regardless of
the file's name.
---- orig/Makefile.in 2006-01-14 08:14:29
-+++ Makefile.in 2005-11-07 04:37:17
-@@ -34,7 +34,7 @@ OBJS1=rsync.o generator.o receiver.o cle
- main.o checksum.o match.o syscall.o log.o backup.o
- OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o \
- fileio.o batch.o clientname.o chmod.o
+--- old/Makefile.in
++++ new/Makefile.in
+@@ -35,7 +35,7 @@ OBJS1=flist.o rsync.o generator.o receiv
+ util.o main.o checksum.o match.o syscall.o log.o backup.o
+ OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o \
+ fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o
+OBJS3=progress.o pipe.o hashlink.o
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
-OBJS3=progress.o pipe.o
+OBJS3=progress.o pipe.o hashlink.o
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
---- orig/hashlink.c 2004-09-24 16:44:25
-+++ hashlink.c 2004-09-24 16:44:25
-@@ -0,0 +1,340 @@
+--- old/flist.c
++++ new/flist.c
+@@ -63,6 +63,7 @@ extern int copy_unsafe_links;
+ extern int protocol_version;
+ extern int sanitize_paths;
+ extern struct stats stats;
++extern char *link_by_hash_dir;
+
+ extern char curr_dir[MAXPATHLEN];
+
+@@ -818,7 +819,7 @@ static struct file_struct *recv_file_ent
+ extra_len += (S_ISDIR(mode) ? 2 : 1) * EXTRA_LEN;
+ #endif
+
+- if (always_checksum && S_ISREG(mode))
++ if ((always_checksum || link_by_hash_dir) && S_ISREG(mode))
+ extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
+
+ if (file_length > 0xFFFFFFFFu && S_ISREG(mode))
+--- old/hashlink.c
++++ new/hashlink.c
+@@ -0,0 +1,336 @@
+ }
+ free(dirname);
+
+ if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
+ rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
+ free(hashname);
+ }
+ free(dirname);
+
+ if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
+ rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
+ free(hashname);
+ } else {
+ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
+ linkname, full_fname(fname));
+ } else {
+ rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
+ linkname, full_fname(fname));
+ rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
+ full_fname(fname),linkname);
+
+ rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
+ full_fname(fname),linkname);
+
+ if (rc != 0) {
+ rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
+ full_fname(fnametmp), full_fname(fname));
+ if (rc != 0) {
+ rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
+ full_fname(fnametmp), full_fname(fname));
char *tmpdir = NULL;
char *partial_dir = NULL;
char *basis_dir[MAX_BASIS_DIRS+1];
+char *link_by_hash_dir = NULL;
char *config_file = NULL;
char *shell_cmd = NULL;
char *tmpdir = NULL;
char *partial_dir = NULL;
char *basis_dir[MAX_BASIS_DIRS+1];
+char *link_by_hash_dir = NULL;
char *config_file = NULL;
char *shell_cmd = NULL;
rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n");
rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n");
rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n");
+ rprintf(F," --link-by-hash=DIR create hardlinks by hash into DIR\n");
rprintf(F," -z, --compress compress file data during the transfer\n");
rprintf(F," --compress-level=NUM explicitly set compression level\n");
rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n");
rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n");
rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n");
+ rprintf(F," --link-by-hash=DIR create hardlinks by hash into DIR\n");
rprintf(F," -z, --compress compress file data during the transfer\n");
rprintf(F," --compress-level=NUM explicitly set compression level\n");
- rprintf(F," -C, --cvs-exclude auto-ignore files the same way CVS does\n");
-@@ -383,7 +385,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OP
+ rprintf(F," --skip-compress=LIST skip compressing files with a suffix in LIST\n");
+@@ -435,7 +437,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OP
OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP,
OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP,
OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
{"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
+ {"link-by-hash", 0, POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
{"fuzzy", 'y', POPT_ARG_NONE, &fuzzy_basis, 0, 0, 0 },
{"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
{"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
+ {"link-by-hash", 0, POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
{"fuzzy", 'y', POPT_ARG_NONE, &fuzzy_basis, 0, 0, 0 },
{"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 },
- {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 'z', 0, 0 },
-@@ -1057,6 +1060,21 @@ int parse_arguments(int *argc, const cha
- usage(FINFO);
- exit_cleanup(0);
+ {"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
+@@ -1205,6 +1208,21 @@ int parse_arguments(int *argc, const cha
+ return 0;
+ #endif
---- orig/receiver.c 2006-01-31 02:30:18
-+++ receiver.c 2005-01-15 21:29:13
-@@ -54,6 +54,7 @@ extern int delay_updates;
- extern struct stats stats;
- extern char *log_format;
- extern char *tmpdir;
-+extern char *link_by_hash_dir;
- extern char *partial_dir;
- extern char *basis_dir[];
- extern struct file_list *the_file_list;
-@@ -186,12 +187,13 @@ static int get_tmpname(char *fnametmp, c
+--- old/receiver.c
++++ new/receiver.c
+@@ -125,12 +125,14 @@ int get_tmpname(char *fnametmp, char *fn
-- char *fname, int fd, OFF_T total_size)
-+ char *fname, int fd, OFF_T total_size, char *md4)
+- const char *fname, int fd, OFF_T total_size)
++ const char *fname, int fd, OFF_T total_size,
++ const char *md4)
+ receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
}
+ receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
}
- static void handle_delayed_updates(struct file_list *flist, char *local_name)
-@@ -666,8 +677,12 @@ int recv_files(int f_in, struct file_lis
- rprintf(FINFO, "%s\n", fname);
+ static void handle_delayed_updates(char *local_name)
+@@ -665,7 +676,7 @@ int recv_files(int f_in, char *local_nam
-- fname, fd2, file->length);
-+ fname, fd2, file->length, file->u.sum);
+- fname, fd2, F_LENGTH(file));
++ fname, fd2, F_LENGTH(file), F_SUM(file));
+
+ log_item(log_code, file, &initial_stats, iflags, NULL);
- if (!log_before_transfer)
- log_item(file, &initial_stats, iflags, NULL);
---- orig/rsync.c 2006-01-31 02:30:18
-+++ rsync.c 2006-01-30 07:19:44
-@@ -40,6 +40,7 @@ extern int inplace;
+--- old/rsync.c
++++ new/rsync.c
+@@ -48,6 +48,7 @@ extern int inplace;
+ extern int flist_eof;
-
-
- /*
-@@ -213,8 +214,15 @@ void finish_transfer(char *fname, char *
+ extern struct file_list *cur_flist, *first_flist, *dir_flist;
+ extern struct chmod_mode_struct *daemon_chmod_modes;
+ #ifdef ICONV_OPTION
+@@ -458,8 +459,15 @@ void finish_transfer(const char *fname,
/* move tmp file over real file */
if (verbose > 2)
rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
- ret = robust_rename(fnametmp, fname, partialptr,
- file->mode & INITACCESSPERMS);
/* move tmp file over real file */
if (verbose > 2)
rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
- ret = robust_rename(fnametmp, fname, partialptr,
- file->mode & INITACCESSPERMS);
if (ret < 0) {
rsyserr(FERROR, errno, "%s %s -> \"%s\"",
ret == -2 ? "copy" : "rename",
if (ret < 0) {
rsyserr(FERROR, errno, "%s %s -> \"%s\"",
ret == -2 ? "copy" : "rename",
- #include "byteorder.h"
---- orig/rsync.yo 2006-01-31 03:05:44
-+++ rsync.yo 2005-02-13 06:58:47
-@@ -361,6 +361,7 @@ to the detailed description below for a
+ #define EMPTY_ITEM_LIST {NULL, 0, 0}
+--- old/rsync.yo
++++ new/rsync.yo
+@@ -392,6 +392,7 @@ to the detailed description below for a
--compare-dest=DIR also compare received files relative to DIR
--copy-dest=DIR ... and include copies of unchanged files
--link-dest=DIR hardlink to files in DIR when unchanged
+ --link-by-hash=DIR create hardlinks by hash into DIR
-z, --compress compress file data during the transfer
--compress-level=NUM explicitly set compression level
--compare-dest=DIR also compare received files relative to DIR
--copy-dest=DIR ... and include copies of unchanged files
--link-dest=DIR hardlink to files in DIR when unchanged
+ --link-by-hash=DIR create hardlinks by hash into DIR
-z, --compress compress file data during the transfer
--compress-level=NUM explicitly set compression level