Commit | Line | Data |
---|---|---|
03019e41 | 1 | Jason M. Felice wrote: |
2eb075b2 WD |
2 | |
3 | This patch adds the --link-by-hash=DIR option, which hard links received | |
4 | files in a link farm arranged by MD4 file hash. The result is that the system | |
5 | will only store one copy of the unique contents of each file, regardless of | |
6 | the file's name. | |
7 | ||
03019e41 WD |
8 | To use this patch, run these commands for a successful build: |
9 | ||
10 | patch -p1 <patches/link-by-hash.diff | |
11 | ./prepare-source | |
12 | ./configure | |
13 | make | |
2eb075b2 | 14 | |
cc3e685d | 15 | diff --git a/Makefile.in b/Makefile.in |
fc557362 | 16 | index feacb90..b27b1e7 100644 |
cc3e685d WD |
17 | --- a/Makefile.in |
18 | +++ b/Makefile.in | |
fc557362 WD |
19 | @@ -37,7 +37,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \ |
20 | util.o main.o checksum.o match.o syscall.o log.o backup.o delete.o | |
790ba11a | 21 | OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \ |
5795bf59 | 22 | fileio.o batch.o clientname.o chmod.o acls.o xattrs.o |
8a529471 WD |
23 | -OBJS3=progress.o pipe.o |
24 | +OBJS3=progress.o pipe.o hashlink.o | |
25 | DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o | |
26 | popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ | |
27 | popt/popthelp.o popt/poptparse.o | |
cc3e685d | 28 | diff --git a/flist.c b/flist.c |
fc557362 | 29 | index 09b4fc5..570bcee 100644 |
cc3e685d WD |
30 | --- a/flist.c |
31 | +++ b/flist.c | |
fc557362 | 32 | @@ -73,6 +73,7 @@ extern int sender_keeps_checksum; |
d4dd2dd5 | 33 | extern int unsort_ndx; |
96148342 | 34 | extern struct stats stats; |
790ba11a | 35 | extern char *filesfrom_host; |
96148342 | 36 | +extern char *link_by_hash_dir; |
fc557362 | 37 | extern char *usermap, *groupmap; |
d4dd2dd5 WD |
38 | |
39 | extern char curr_dir[MAXPATHLEN]; | |
fc557362 | 40 | @@ -881,7 +882,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, |
c0c7984e | 41 | extra_len += EXTRA_LEN; |
96148342 WD |
42 | #endif |
43 | ||
44 | - if (always_checksum && S_ISREG(mode)) | |
45 | + if ((always_checksum || link_by_hash_dir) && S_ISREG(mode)) | |
46 | extra_len += SUM_EXTRA_CNT * EXTRA_LEN; | |
47 | ||
7f0bf1cb | 48 | #if SIZEOF_INT64 >= 8 |
cc3e685d WD |
49 | diff --git a/hashlink.c b/hashlink.c |
50 | new file mode 100644 | |
fc557362 | 51 | index 0000000..15e2a73 |
cc3e685d WD |
52 | --- /dev/null |
53 | +++ b/hashlink.c | |
fc557362 | 54 | @@ -0,0 +1,339 @@ |
c57f4101 WD |
55 | +/* |
56 | + Copyright (C) Cronosys, LLC 2004 | |
57 | + | |
58 | + This program is free software; you can redistribute it and/or modify | |
59 | + it under the terms of the GNU General Public License as published by | |
60 | + the Free Software Foundation; either version 2 of the License, or | |
61 | + (at your option) any later version. | |
62 | + | |
63 | + This program is distributed in the hope that it will be useful, | |
64 | + but WITHOUT ANY WARRANTY; without even the implied warranty of | |
65 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
66 | + GNU General Public License for more details. | |
67 | + | |
68 | + You should have received a copy of the GNU General Public License | |
69 | + along with this program; if not, write to the Free Software | |
70 | + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
71 | +*/ | |
72 | + | |
73 | +/* This file contains code used by the --link-by-hash option. */ | |
74 | + | |
75 | +#include "rsync.h" | |
76 | + | |
77 | +extern char *link_by_hash_dir; | |
78 | + | |
96148342 | 79 | +#ifdef HAVE_LINK |
c57f4101 | 80 | + |
96148342 | 81 | +char *make_hash_name(struct file_struct *file) |
c57f4101 WD |
82 | +{ |
83 | + char hash[33], *dst; | |
96148342 | 84 | + uchar c, *src = (uchar*)F_SUM(file); |
c57f4101 WD |
85 | + int i; |
86 | + | |
c57f4101 WD |
87 | + for (dst = hash, i = 0; i < 4; i++, src++) { |
88 | + c = *src >> 4; | |
89 | + *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0'); | |
90 | + c = *src & 0x0f; | |
91 | + *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0'); | |
92 | + } | |
93 | + *dst++ = '/'; | |
94 | + for (i = 0; i < 12; i++, src++) { | |
95 | + c = *src >> 4; | |
96 | + *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0'); | |
97 | + c = *src & 0x0f; | |
98 | + *(dst++) = (c >= 10) ? (c - 10 + 'a') : (c + '0'); | |
99 | + } | |
100 | + *dst = 0; | |
101 | + | |
fc557362 WD |
102 | + if (asprintf(&dst,"%s/%s",link_by_hash_dir,hash) < 0) |
103 | + out_of_memory("make_hash_name"); | |
c57f4101 WD |
104 | + return dst; |
105 | +} | |
106 | + | |
107 | + | |
108 | +void kill_hashfile(struct hashfile_struct *hashfile) | |
109 | +{ | |
110 | + if (!hashfile) | |
111 | + return; | |
112 | + free(hashfile->name); | |
113 | + close(hashfile->fd); | |
114 | + free(hashfile); | |
115 | +} | |
116 | + | |
117 | + | |
118 | +void kill_hashfiles(struct hashfile_struct *hashfiles) | |
119 | +{ | |
120 | + struct hashfile_struct *iter, *next; | |
121 | + if ((iter = hashfiles) != NULL) { | |
122 | + do { | |
123 | + next = iter->next; | |
124 | + kill_hashfile(iter); | |
125 | + iter = next; | |
126 | + } while (iter != hashfiles); | |
127 | + } | |
128 | +} | |
129 | + | |
130 | + | |
131 | +struct hashfile_struct *find_hashfiles(char *hashname, int64 size, long *fnbr) | |
132 | +{ | |
133 | + DIR *d; | |
134 | + struct dirent *di; | |
135 | + struct hashfile_struct *hashfiles = NULL, *hashfile; | |
136 | + STRUCT_STAT st; | |
137 | + long this_fnbr; | |
138 | + | |
139 | + *fnbr = 0; | |
47841496 | 140 | + |
c57f4101 WD |
141 | + /* Build a list of potential candidates and open |
142 | + * them. */ | |
143 | + if ((d = opendir(hashname)) == NULL) { | |
d0320a46 | 144 | + rsyserr(FERROR, errno, "opendir failed: \"%s\"", hashname); |
c57f4101 WD |
145 | + free(hashname); |
146 | + return NULL; | |
147 | + } | |
148 | + while ((di = readdir(d)) != NULL) { | |
149 | + if (!strcmp(di->d_name,".") || !strcmp(di->d_name,"..")) { | |
150 | + continue; | |
151 | + } | |
152 | + | |
153 | + /* We need to have the largest fnbr in case we need to store | |
154 | + * a new file. */ | |
155 | + this_fnbr = atol(di->d_name); | |
156 | + if (this_fnbr > *fnbr) | |
157 | + *fnbr = this_fnbr; | |
158 | + | |
39cc637d | 159 | + hashfile = new_array(struct hashfile_struct, 1); |
fc557362 WD |
160 | + if (asprintf(&hashfile->name,"%s/%s",hashname, di->d_name) < 0) |
161 | + out_of_memory("find_hashfiles"); | |
c57f4101 | 162 | + if (do_stat(hashfile->name,&st) == -1) { |
d0320a46 | 163 | + rsyserr(FERROR, errno, "stat failed: %s", hashfile->name); |
c57f4101 WD |
164 | + kill_hashfile(hashfile); |
165 | + continue; | |
166 | + } | |
167 | + if (st.st_size != size) { | |
168 | + kill_hashfile(hashfile); | |
169 | + continue; | |
170 | + } | |
171 | + hashfile->nlink = st.st_nlink; | |
172 | + hashfile->fd = open(hashfile->name,O_RDONLY|O_BINARY); | |
173 | + if (hashfile->fd == -1) { | |
d0320a46 | 174 | + rsyserr(FERROR, errno, "open failed: %s", hashfile->name); |
c57f4101 WD |
175 | + kill_hashfile(hashfile); |
176 | + continue; | |
177 | + } | |
178 | + if (hashfiles == NULL) | |
179 | + hashfiles = hashfile->next = hashfile->prev = hashfile; | |
180 | + else { | |
181 | + hashfile->next = hashfiles; | |
182 | + hashfile->prev = hashfiles->prev; | |
183 | + hashfile->next->prev = hashfile; | |
184 | + hashfile->prev->next = hashfile; | |
185 | + } | |
186 | + } | |
187 | + closedir(d); | |
188 | + | |
189 | + return hashfiles; | |
190 | +} | |
191 | + | |
192 | + | |
193 | +struct hashfile_struct *compare_hashfiles(int fd,struct hashfile_struct *files) | |
194 | +{ | |
195 | + int amt, hamt; | |
196 | + char buffer[BUFSIZ], cmpbuffer[BUFSIZ]; | |
197 | + struct hashfile_struct *iter, *next, *best; | |
198 | + uint32 nlink; | |
199 | + | |
200 | + if (!files) | |
201 | + return NULL; | |
202 | + | |
203 | + iter = files; /* in case files are 0 bytes */ | |
204 | + while ((amt = read(fd, buffer, BUFSIZ)) > 0) { | |
205 | + iter = files; | |
206 | + do { | |
207 | + /* Icky bit to resync when we steal the first node. */ | |
208 | + if (!files) | |
209 | + files = iter; | |
210 | + | |
211 | + next = iter->next; | |
212 | + | |
213 | + hamt = read(iter->fd, cmpbuffer, BUFSIZ); | |
214 | + if (amt != hamt || memcmp(buffer, cmpbuffer, amt)) { | |
215 | + if (iter == files) { | |
216 | + files = files->prev; | |
217 | + } | |
218 | + if (iter->next == iter) { | |
219 | + files = next = NULL; | |
220 | + } else { | |
221 | + next = iter->next; | |
222 | + if (iter == files) { | |
223 | + /* So we know to resync */ | |
224 | + files = NULL; | |
225 | + } | |
226 | + } | |
227 | + iter->next->prev = iter->prev; | |
228 | + iter->prev->next = iter->next; | |
229 | + kill_hashfile(iter); | |
230 | + } | |
231 | + | |
232 | + iter = next; | |
233 | + } while (iter != files); | |
234 | + | |
235 | + if (iter == NULL && files == NULL) { | |
236 | + /* There are no matches. */ | |
237 | + return NULL; | |
238 | + } | |
c57f4101 WD |
239 | + } |
240 | + | |
241 | + if (amt == -1) { | |
d0320a46 | 242 | + rsyserr(FERROR, errno, "read failed in compare_hashfiles()"); |
c57f4101 WD |
243 | + kill_hashfiles(files); |
244 | + return NULL; | |
245 | + } | |
246 | + | |
247 | + /* If we only have one file left, use it. */ | |
248 | + if (files == files->next) { | |
249 | + return files; | |
250 | + } | |
251 | + | |
252 | + /* All files which remain in the list are identical and should have | |
253 | + * the same size. We pick the one with the lowest link count (we | |
254 | + * may have rolled over because we hit the maximum link count for | |
255 | + * the filesystem). */ | |
256 | + best = iter = files; | |
257 | + nlink = iter->nlink; | |
258 | + do { | |
259 | + if (iter->nlink < nlink) { | |
260 | + nlink = iter->nlink; | |
261 | + best = iter; | |
262 | + } | |
263 | + iter = iter->next; | |
264 | + } while (iter != files); | |
265 | + | |
266 | + best->next->prev = best->prev; | |
267 | + best->prev->next = best->next; | |
268 | + if (files == best) | |
269 | + files = files->next; | |
270 | + kill_hashfiles(files); | |
271 | + return best; | |
272 | +} | |
273 | + | |
274 | + | |
7bfcb297 | 275 | +int link_by_hash(const char *fnametmp, const char *fname, struct file_struct *file) |
c57f4101 WD |
276 | +{ |
277 | + STRUCT_STAT st; | |
47841496 | 278 | + char *hashname = make_hash_name(file); |
c57f4101 WD |
279 | + int first = 0, rc; |
280 | + char *linkname; | |
281 | + long last_fnbr; | |
282 | + | |
a3044834 | 283 | + if (F_LENGTH(file) == 0) |
4c38ad2a | 284 | + return robust_rename(fnametmp, fname, NULL, 0644); |
c57f4101 WD |
285 | + |
286 | + if (do_stat(hashname, &st) == -1) { | |
287 | + char *dirname; | |
288 | + | |
289 | + /* Directory does not exist. */ | |
290 | + dirname = strdup(hashname); | |
291 | + *strrchr(dirname,'/') = 0; | |
292 | + if (do_mkdir(dirname, 0755) == -1 && errno != EEXIST) { | |
d0320a46 | 293 | + rsyserr(FERROR, errno, "mkdir failed: %s", dirname); |
c57f4101 WD |
294 | + free(hashname); |
295 | + free(dirname); | |
4c38ad2a | 296 | + return robust_rename(fnametmp, fname, NULL, 0644); |
c57f4101 WD |
297 | + } |
298 | + free(dirname); | |
299 | + | |
300 | + if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) { | |
d0320a46 | 301 | + rsyserr(FERROR, errno, "mkdir failed: %s", hashname); |
c57f4101 | 302 | + free(hashname); |
4c38ad2a | 303 | + return robust_rename(fnametmp, fname, NULL, 0644); |
c57f4101 WD |
304 | + } |
305 | + | |
306 | + first = 1; | |
fc557362 WD |
307 | + if (asprintf(&linkname,"%s/0",hashname) < 0) |
308 | + out_of_memory("link_by_hash"); | |
c57f4101 | 309 | + rprintf(FINFO, "(1) linkname = %s\n", linkname); |
c57f4101 WD |
310 | + } else { |
311 | + struct hashfile_struct *hashfiles, *hashfile; | |
c57f4101 WD |
312 | + |
313 | + if (do_stat(fnametmp,&st) == -1) { | |
d0320a46 | 314 | + rsyserr(FERROR, errno, "stat failed: %s", fname); |
c57f4101 WD |
315 | + return -1; |
316 | + } | |
317 | + hashfiles = find_hashfiles(hashname, st.st_size, &last_fnbr); | |
318 | + | |
319 | + if (hashfiles == NULL) { | |
320 | + first = 1; | |
fc557362 WD |
321 | + if (asprintf(&linkname,"%s/0",hashname) < 0) |
322 | + out_of_memory("link_by_hash"); | |
c57f4101 WD |
323 | + rprintf(FINFO, "(2) linkname = %s\n", linkname); |
324 | + } else { | |
47841496 | 325 | + int fd; |
c57f4101 WD |
326 | + /* Search for one identical to us. */ |
327 | + if ((fd = open(fnametmp,O_RDONLY|O_BINARY)) == -1) { | |
d0320a46 | 328 | + rsyserr(FERROR, errno, "open failed: %s", fnametmp); |
c57f4101 WD |
329 | + kill_hashfiles(hashfiles); |
330 | + return -1; | |
331 | + } | |
332 | + hashfile = compare_hashfiles(fd, hashfiles); | |
333 | + hashfiles = NULL; | |
47841496 | 334 | + close(fd); |
c57f4101 WD |
335 | + |
336 | + if (hashfile) { | |
337 | + first = 0; | |
338 | + linkname = strdup(hashfile->name); | |
339 | + rprintf(FINFO, "(3) linkname = %s\n", linkname); | |
340 | + kill_hashfile(hashfile); | |
341 | + } else { | |
342 | + first = 1; | |
fc557362 WD |
343 | + if (asprintf(&linkname, "%s/%ld", hashname, last_fnbr + 1) < 0) |
344 | + out_of_memory("link_by_hash"); | |
c57f4101 WD |
345 | + rprintf(FINFO, "(4) linkname = %s\n", linkname); |
346 | + } | |
347 | + } | |
348 | + } | |
349 | + | |
350 | + if (!first) { | |
351 | + rprintf(FINFO, "link-by-hash (existing): \"%s\" -> %s\n", | |
352 | + linkname, full_fname(fname)); | |
cad12f62 | 353 | + robust_unlink(fname); |
c57f4101 WD |
354 | + rc = do_link(linkname, fname); |
355 | + if (rc == -1) { | |
356 | + if (errno == EMLINK) { | |
357 | + first = 1; | |
358 | + free(linkname); | |
fc557362 WD |
359 | + if (asprintf(&linkname,"%s/%ld",hashname, last_fnbr + 1) < 0) |
360 | + out_of_memory("link_by_hash"); | |
c57f4101 WD |
361 | + rprintf(FINFO, "(5) linkname = %s\n", linkname); |
362 | + rprintf(FINFO,"link-by-hash: max link count exceeded, starting new file \"%s\".\n", linkname); | |
363 | + } else { | |
fe6407b5 WD |
364 | + rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", |
365 | + linkname, full_fname(fname)); | |
4c38ad2a | 366 | + rc = robust_rename(fnametmp, fname, NULL, 0644); |
c57f4101 WD |
367 | + } |
368 | + } else { | |
369 | + do_unlink(fnametmp); | |
370 | + } | |
371 | + } | |
372 | + | |
373 | + if (first) { | |
374 | + rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n", | |
375 | + full_fname(fname),linkname); | |
376 | + | |
4c38ad2a | 377 | + rc = robust_rename(fnametmp, fname, NULL, 0644); |
c57f4101 | 378 | + if (rc != 0) { |
fe6407b5 WD |
379 | + rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"", |
380 | + full_fname(fnametmp), full_fname(fname)); | |
c57f4101 WD |
381 | + } |
382 | + rc = do_link(fname,linkname); | |
383 | + if (rc != 0) { | |
fe6407b5 WD |
384 | + rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", |
385 | + full_fname(fname), linkname); | |
c57f4101 WD |
386 | + } |
387 | + } | |
388 | + | |
389 | + free(linkname); | |
390 | + free(hashname); | |
391 | + return rc; | |
392 | +} | |
c57f4101 | 393 | +#endif |
cc3e685d | 394 | diff --git a/options.c b/options.c |
fc557362 | 395 | index e7c6c61..73b1aa4 100644 |
cc3e685d WD |
396 | --- a/options.c |
397 | +++ b/options.c | |
fc557362 | 398 | @@ -158,6 +158,7 @@ char *backup_suffix = NULL; |
e0e47893 WD |
399 | char *tmpdir = NULL; |
400 | char *partial_dir = NULL; | |
401 | char *basis_dir[MAX_BASIS_DIRS+1]; | |
c57f4101 | 402 | +char *link_by_hash_dir = NULL; |
e0e47893 WD |
403 | char *config_file = NULL; |
404 | char *shell_cmd = NULL; | |
ff318e90 | 405 | char *logfile_name = NULL; |
fc557362 | 406 | @@ -745,6 +746,7 @@ void usage(enum logcode F) |
c57f4101 | 407 | rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n"); |
6ba1be7d | 408 | rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n"); |
0808daa5 | 409 | rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n"); |
b78a6aba WD |
410 | + rprintf(F," --link-by-hash=DIR create hardlinks by hash into DIR\n"); |
411 | rprintf(F," -z, --compress compress file data during the transfer\n"); | |
610969d1 | 412 | rprintf(F," --compress-level=NUM explicitly set compression level\n"); |
5ba66156 | 413 | rprintf(F," --skip-compress=LIST skip compressing files with a suffix in LIST\n"); |
fc557362 | 414 | @@ -798,7 +800,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, |
0ca6aebe | 415 | OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, |
5398d042 | 416 | OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, |
fc557362 WD |
417 | OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, |
418 | - OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, | |
419 | + OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_LINK_BY_HASH, | |
0ca6aebe | 420 | OPT_SERVER, OPT_REFUSED_BASE = 9000}; |
c57f4101 | 421 | |
70c5d149 | 422 | static struct poptOption long_options[] = { |
fc557362 | 423 | @@ -937,6 +939,7 @@ static struct poptOption long_options[] = { |
0808daa5 | 424 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, |
6ba1be7d | 425 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, |
0808daa5 | 426 | {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 }, |
5388f859 | 427 | + {"link-by-hash", 0, POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0}, |
c0c7984e WD |
428 | {"fuzzy", 'y', POPT_ARG_VAL, &fuzzy_basis, 1, 0, 0 }, |
429 | {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, | |
430 | {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, | |
fc557362 | 431 | @@ -1742,6 +1745,21 @@ int parse_arguments(int *argc_p, const char ***argv_p) |
5795bf59 | 432 | return 0; |
ffc18846 WD |
433 | #endif |
434 | ||
c57f4101 | 435 | + case OPT_LINK_BY_HASH: |
96148342 | 436 | +#ifdef HAVE_LINK |
d0320a46 WD |
437 | + arg = poptGetOptArg(pc); |
438 | + if (sanitize_paths) | |
c0c7984e | 439 | + arg = sanitize_path(NULL, arg, NULL, 0, SP_DEFAULT); |
d0320a46 | 440 | + link_by_hash_dir = (char *)arg; |
c57f4101 WD |
441 | + break; |
442 | +#else | |
443 | + snprintf(err_buf, sizeof err_buf, | |
444 | + "hard links are not supported on this %s\n", | |
445 | + am_server ? "server" : "client"); | |
446 | + rprintf(FERROR, "ERROR: %s", err_buf); | |
447 | + return 0; | |
448 | +#endif | |
449 | + | |
450 | default: | |
451 | /* A large opt value means that set_refuse_options() | |
27a7053c | 452 | * turned this option off. */ |
fc557362 | 453 | @@ -2584,6 +2602,11 @@ void server_options(char **args, int *argc_p) |
5ba66156 WD |
454 | } else if (inplace) |
455 | args[ac++] = "--inplace"; | |
7b675ff5 | 456 | |
c57f4101 WD |
457 | + if (link_by_hash_dir && am_sender) { |
458 | + args[ac++] = "--link-by-hash"; | |
459 | + args[ac++] = link_by_hash_dir; | |
7b675ff5 WD |
460 | + } |
461 | + | |
def2ace9 WD |
462 | if (files_from && (!am_sender || filesfrom_host)) { |
463 | if (filesfrom_host) { | |
7b675ff5 | 464 | args[ac++] = "--files-from"; |
cc3e685d | 465 | diff --git a/receiver.c b/receiver.c |
fc557362 | 466 | index 4325e30..2709d5e 100644 |
cc3e685d WD |
467 | --- a/receiver.c |
468 | +++ b/receiver.c | |
fc557362 | 469 | @@ -164,11 +164,13 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file) |
99397a25 | 470 | } |
c57f4101 | 471 | |
dc3ae351 | 472 | static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
87a38eea | 473 | - const char *fname, int fd, OFF_T total_size) |
a3044834 WD |
474 | + const char *fname, int fd, OFF_T total_size, |
475 | + const char *md4) | |
c57f4101 | 476 | { |
87a38eea | 477 | static char file_sum1[MAX_DIGEST_LEN]; |
dc3ae351 | 478 | struct map_struct *mapbuf; |
c57f4101 | 479 | struct sum_struct sum; |
87a38eea | 480 | + md_context mdfour_data; |
fc557362 | 481 | int32 len; |
c57f4101 WD |
482 | OFF_T offset = 0; |
483 | OFF_T offset2; | |
963ca808 | 484 | @@ -188,6 +190,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
dc3ae351 WD |
485 | } else |
486 | mapbuf = NULL; | |
7b675ff5 | 487 | |
c57f4101 WD |
488 | + if (md4) |
489 | + mdfour_begin(&mdfour_data); | |
7b675ff5 WD |
490 | + |
491 | sum_init(checksum_seed); | |
c57f4101 | 492 | |
fc068916 | 493 | if (append_mode > 0) { |
963ca808 | 494 | @@ -232,6 +237,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
c57f4101 WD |
495 | cleanup_got_literal = 1; |
496 | ||
2c2d83dc | 497 | sum_update(data, i); |
c57f4101 | 498 | + if (md4) |
4c38ad2a | 499 | + mdfour_update(&mdfour_data, (uchar*)data, i); |
c57f4101 | 500 | |
afbebe13 WD |
501 | if (fd != -1 && write_file(fd,data,i) != i) |
502 | goto report_write_error; | |
963ca808 | 503 | @@ -258,6 +265,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
c57f4101 WD |
504 | |
505 | see_token(map, len); | |
2c2d83dc | 506 | sum_update(map, len); |
c57f4101 | 507 | + if (md4) |
4c38ad2a | 508 | + mdfour_update(&mdfour_data, (uchar*)map, len); |
c57f4101 WD |
509 | } |
510 | ||
d4dd2dd5 | 511 | if (updating_basis_or_equiv) { |
fc557362 WD |
512 | @@ -305,6 +314,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, |
513 | if (sum_end(file_sum1) != checksum_len) | |
514 | overflow_exit("checksum_len"); /* Impossible... */ | |
c57f4101 | 515 | |
c57f4101 | 516 | + if (md4) |
87a38eea | 517 | + mdfour_result(&mdfour_data, (uchar*)md4); |
fc557362 | 518 | + |
dc3ae351 WD |
519 | if (mapbuf) |
520 | unmap_file(mapbuf); | |
fc557362 WD |
521 | |
522 | @@ -319,7 +331,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, | |
5823d322 WD |
523 | |
524 | static void discard_receive_data(int f_in, OFF_T length) | |
525 | { | |
dc3ae351 WD |
526 | - receive_data(f_in, NULL, -1, 0, NULL, -1, length); |
527 | + receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL); | |
5823d322 WD |
528 | } |
529 | ||
fc068916 | 530 | static void handle_delayed_updates(char *local_name) |
fc557362 | 531 | @@ -744,7 +756,7 @@ int recv_files(int f_in, char *local_name) |
a3044834 WD |
532 | |
533 | /* recv file data */ | |
534 | recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, | |
535 | - fname, fd2, F_LENGTH(file)); | |
536 | + fname, fd2, F_LENGTH(file), F_SUM(file)); | |
537 | ||
538 | log_item(log_code, file, &initial_stats, iflags, NULL); | |
539 | ||
cc3e685d | 540 | diff --git a/rsync.c b/rsync.c |
fc557362 | 541 | index 2c026a2..87f6d54 100644 |
cc3e685d WD |
542 | --- a/rsync.c |
543 | +++ b/rsync.c | |
fc557362 WD |
544 | @@ -48,6 +48,7 @@ extern int flist_eof; |
545 | extern int msgs2stderr; | |
4c107044 | 546 | extern int keep_dirlinks; |
c57f4101 | 547 | extern int make_backups; |
c968d24c | 548 | +extern char *link_by_hash_dir; |
fc068916 | 549 | extern struct file_list *cur_flist, *first_flist, *dir_flist; |
c769ea2c | 550 | extern struct chmod_mode_struct *daemon_chmod_modes; |
5ba66156 | 551 | #ifdef ICONV_OPTION |
fc557362 | 552 | @@ -575,8 +576,15 @@ int finish_transfer(const char *fname, const char *fnametmp, |
93ca4d27 | 553 | /* move tmp file over real file */ |
fc557362 | 554 | if (DEBUG_GTE(RECV, 1)) |
93ca4d27 | 555 | rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname); |
4c15e800 | 556 | - ret = robust_rename(fnametmp, fname, temp_copy_name, |
4a65fe72 | 557 | - file->mode & INITACCESSPERMS); |
96148342 | 558 | +#ifdef HAVE_LINK |
2eb075b2 | 559 | + if (link_by_hash_dir) |
7b675ff5 | 560 | + ret = link_by_hash(fnametmp, fname, file); |
2eb075b2 | 561 | + else |
c57f4101 | 562 | +#endif |
4a65fe72 | 563 | + { |
4c15e800 | 564 | + ret = robust_rename(fnametmp, fname, temp_copy_name, |
4a65fe72 WD |
565 | + file->mode & INITACCESSPERMS); |
566 | + } | |
54691942 | 567 | if (ret < 0) { |
cc3e685d | 568 | rsyserr(FERROR_XFER, errno, "%s %s -> \"%s\"", |
93ca4d27 | 569 | ret == -2 ? "copy" : "rename", |
cc3e685d | 570 | diff --git a/rsync.h b/rsync.h |
fc557362 | 571 | index be7cf8a..d4e2aca 100644 |
cc3e685d WD |
572 | --- a/rsync.h |
573 | +++ b/rsync.h | |
fc557362 WD |
574 | @@ -853,6 +853,14 @@ struct stats { |
575 | int xferred_files; | |
c57f4101 WD |
576 | }; |
577 | ||
578 | +struct hashfile_struct { | |
579 | + struct hashfile_struct *next; | |
580 | + struct hashfile_struct *prev; | |
581 | + char *name; | |
582 | + int fd; | |
583 | + uint32 nlink; | |
584 | +}; | |
585 | + | |
4a65fe72 | 586 | struct chmod_mode_struct; |
c57f4101 | 587 | |
963ca808 | 588 | struct flist_ndx_item { |
cc3e685d | 589 | diff --git a/rsync.yo b/rsync.yo |
fc557362 | 590 | index 941f7a5..568b481 100644 |
cc3e685d WD |
591 | --- a/rsync.yo |
592 | +++ b/rsync.yo | |
fc557362 | 593 | @@ -400,6 +400,7 @@ to the detailed description below for a complete description. verb( |
79f132a1 | 594 | --compare-dest=DIR also compare received files relative to DIR |
6ba1be7d | 595 | --copy-dest=DIR ... and include copies of unchanged files |
79f132a1 | 596 | --link-dest=DIR hardlink to files in DIR when unchanged |
b78a6aba WD |
597 | + --link-by-hash=DIR create hardlinks by hash into DIR |
598 | -z, --compress compress file data during the transfer | |
610969d1 | 599 | --compress-level=NUM explicitly set compression level |
5ba66156 | 600 | --skip-compress=LIST skip compressing files with suffix in LIST |