| 1 | This patch is the start of storing/using checksum information from |
| 2 | extended attribute values. The rsync code only reads the values |
| 3 | at the moment. There is also a perl script that can create them. |
| 4 | |
| 5 | To use this patch, run these commands for a successful build: |
| 6 | |
| 7 | patch -p1 <patches/checksum-xattr.diff |
| 8 | ./configure (optional if already run) |
| 9 | make |
| 10 | |
| 11 | based-on: 3b8f8192227b14e708bf535072485e50f4362270 |
| 12 | diff --git a/flist.c b/flist.c |
| 13 | --- a/flist.c |
| 14 | +++ b/flist.c |
| 15 | @@ -1309,7 +1309,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, |
| 16 | #endif |
| 17 | |
| 18 | if (always_checksum && am_sender && S_ISREG(st.st_mode)) { |
| 19 | - file_checksum(thisname, tmp_sum, st.st_size); |
| 20 | + if (!get_sum_xattr(thisname, &st, tmp_sum)) |
| 21 | + file_checksum(thisname, tmp_sum, st.st_size); |
| 22 | if (sender_keeps_checksum) |
| 23 | extra_len += SUM_EXTRA_CNT * EXTRA_LEN; |
| 24 | } |
| 25 | diff --git a/generator.c b/generator.c |
| 26 | --- a/generator.c |
| 27 | +++ b/generator.c |
| 28 | @@ -532,7 +532,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) |
| 29 | of the file time to determine whether to sync */ |
| 30 | if (always_checksum > 0 && S_ISREG(st->st_mode)) { |
| 31 | char sum[MAX_DIGEST_LEN]; |
| 32 | - file_checksum(fn, sum, st->st_size); |
| 33 | + if (!get_sum_xattr(fn, st, sum)) |
| 34 | + file_checksum(fn, sum, st->st_size); |
| 35 | return memcmp(sum, F_SUM(file), checksum_len) == 0; |
| 36 | } |
| 37 | |
| 38 | diff --git a/support/xsums b/support/xsums |
| 39 | new file mode 100755 |
| 40 | --- /dev/null |
| 41 | +++ b/support/xsums |
| 42 | @@ -0,0 +1,204 @@ |
| 43 | +#!/usr/bin/perl -w |
| 44 | +use strict; |
| 45 | + |
| 46 | +use Getopt::Long; |
| 47 | +use Cwd qw(abs_path cwd); |
| 48 | +use Digest::MD4; |
| 49 | +use Digest::MD5; |
| 50 | +use File::ExtAttr ':all'; |
| 51 | + |
| 52 | +&Getopt::Long::Configure('bundling'); |
| 53 | +&usage if !&GetOptions( |
| 54 | + 'recurse|r' => \( my $recurse_opt ), |
| 55 | + 'list|l' => \( my $list_opt ), |
| 56 | + 'check|c' => \( my $check_opt ), |
| 57 | + 'verbose|v+' => \( my $verbosity = 0 ), |
| 58 | + 'help|h' => \( my $help_opt ), |
| 59 | +); |
| 60 | +&usage if $help_opt; |
| 61 | + |
| 62 | +my $start_dir = cwd(); |
| 63 | + |
| 64 | +my @dirs = @ARGV; |
| 65 | +@dirs = '.' unless @dirs; |
| 66 | +foreach (@dirs) { |
| 67 | + $_ = abs_path($_); |
| 68 | +} |
| 69 | + |
| 70 | +$| = 1; |
| 71 | + |
| 72 | +my $exit_code = 0; |
| 73 | + |
| 74 | +my $md4 = Digest::MD4->new; |
| 75 | +my $md5 = Digest::MD5->new; |
| 76 | + |
| 77 | +while (@dirs) { |
| 78 | + my $dir = shift @dirs; |
| 79 | + |
| 80 | + if (!chdir($dir)) { |
| 81 | + warn "Unable to chdir to $dir: $!\n"; |
| 82 | + next; |
| 83 | + } |
| 84 | + if (!opendir(DP, '.')) { |
| 85 | + warn "Unable to opendir $dir: $!\n"; |
| 86 | + next; |
| 87 | + } |
| 88 | + |
| 89 | + my $reldir = $dir; |
| 90 | + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; |
| 91 | + print "$reldir ... " if $verbosity; |
| 92 | + |
| 93 | + my @subdirs; |
| 94 | + my $d_cnt = 0; |
| 95 | + my $need_newline = $verbosity; |
| 96 | + while (defined(my $fn = readdir(DP))) { |
| 97 | + next if $fn =~ /^\.\.?$/ || -l $fn; |
| 98 | + if (-d _) { |
| 99 | + push(@subdirs, "$dir/$fn"); |
| 100 | + next; |
| 101 | + } |
| 102 | + next unless -f _; |
| 103 | + $d_cnt++; |
| 104 | + |
| 105 | + my($size,$mtime,$ctime) = (stat(_))[7,9,10]; |
| 106 | + |
| 107 | + my $xsum4 = getfattr($fn, 'rsync.%md4'); |
| 108 | + my $xsum5 = getfattr($fn, 'rsync.%md5'); |
| 109 | + |
| 110 | + my $sum_count = 0; |
| 111 | + foreach ($xsum4, $xsum5) { |
| 112 | + if (defined $_) { |
| 113 | + if (length($_) == 24) { |
| 114 | + my($sz,$mt,$sum) = unpack('V2a16', $_); |
| 115 | + if ($sz != ($size & 0xFFFFFFFF) |
| 116 | + || $mt != ($mtime & 0xFFFFFFFF)) { |
| 117 | + $_ = undef; |
| 118 | + } else { |
| 119 | + $_ = $sum; |
| 120 | + $sum_count++; |
| 121 | + } |
| 122 | + } else { |
| 123 | + $_ = undef; |
| 124 | + } |
| 125 | + } |
| 126 | + } |
| 127 | + |
| 128 | + if ($list_opt) { |
| 129 | + if ($need_newline) { |
| 130 | + print "\n"; |
| 131 | + $need_newline = 0; |
| 132 | + } |
| 133 | + if (defined $xsum4) { |
| 134 | + print ' ', unpack('H32', $xsum4); |
| 135 | + } else { |
| 136 | + print ' ' x (1 + 32); |
| 137 | + } |
| 138 | + if (defined $xsum5) { |
| 139 | + print ' ', unpack('H32', $xsum5); |
| 140 | + } else { |
| 141 | + print ' ' x (1 + 32); |
| 142 | + } |
| 143 | + print $verbosity ? ' ' : " $reldir/"; |
| 144 | + print $fn, "\n"; |
| 145 | + next; |
| 146 | + } |
| 147 | + |
| 148 | + if ($check_opt) { |
| 149 | + if (!$sum_count) { |
| 150 | + if ($need_newline) { |
| 151 | + print "\n"; |
| 152 | + $need_newline = 0; |
| 153 | + } |
| 154 | + print ' ' x (1 + 32 + 1 + 32) if $verbosity > 2; |
| 155 | + print $verbosity ? ' ' : "$reldir/"; |
| 156 | + print $fn, " MISSING\n"; |
| 157 | + next; |
| 158 | + } |
| 159 | + } else { |
| 160 | + next if $sum_count == 2; |
| 161 | + print 'UPDATING' if $need_newline && $verbosity == 1; |
| 162 | + } |
| 163 | + |
| 164 | + if ($need_newline && (!$check_opt || $verbosity > 1)) { |
| 165 | + print "\n"; |
| 166 | + $need_newline = 0; |
| 167 | + } |
| 168 | + |
| 169 | + if (!open(IN, $fn)) { |
| 170 | + print STDERR "Unable to read $fn: $!\n"; |
| 171 | + next; |
| 172 | + } |
| 173 | + |
| 174 | + my($sum4, $sum5); |
| 175 | + while (1) { |
| 176 | + while (sysread(IN, $_, 64*1024)) { |
| 177 | + $md4->add($_); |
| 178 | + $md5->add($_); |
| 179 | + } |
| 180 | + $sum4 = $md4->digest; |
| 181 | + $sum5 = $md5->digest; |
| 182 | + print ' ', unpack('H32', $sum4), ' ', unpack('H32', $sum5) if $verbosity > 2; |
| 183 | + print " $fn" if $verbosity > 1; |
| 184 | + my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10]; |
| 185 | + last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2; |
| 186 | + $size = $size2; |
| 187 | + $mtime = $mtime2; |
| 188 | + $ctime = $ctime2; |
| 189 | + sysseek(IN, 0, 0); |
| 190 | + print " REREADING\n" if $verbosity > 1; |
| 191 | + } |
| 192 | + |
| 193 | + close IN; |
| 194 | + |
| 195 | + if ($check_opt) { |
| 196 | + if ((!defined $xsum4 || $xsum4 eq $sum4) && (!defined $xsum5 || $xsum5 eq $sum5)) { |
| 197 | + print " OK\n" if $verbosity > 1; |
| 198 | + next; |
| 199 | + } |
| 200 | + if ($need_newline) { |
| 201 | + print "\n"; |
| 202 | + $need_newline = 0; |
| 203 | + } |
| 204 | + if ($verbosity < 2) { |
| 205 | + print $verbosity ? ' ' : "$reldir/"; |
| 206 | + print $fn; |
| 207 | + } |
| 208 | + print " FAILED\n"; |
| 209 | + $exit_code = 1; |
| 210 | + } else { |
| 211 | + print "\n" if $verbosity > 1; |
| 212 | + my $szmt = pack('V2', $size, $mtime); # 32-bits, may truncate |
| 213 | + setfattr($fn, 'rsync.%md4', $szmt.$sum4); |
| 214 | + setfattr($fn, 'rsync.%md5', $szmt.$sum5); |
| 215 | + #utime $mtime, $mtime, $fn; # Set mtime if it changes. |
| 216 | + } |
| 217 | + } |
| 218 | + |
| 219 | + if ($need_newline) { |
| 220 | + if ($d_cnt) { |
| 221 | + print "ok\n"; |
| 222 | + } else { |
| 223 | + print "empty\n"; |
| 224 | + } |
| 225 | + } |
| 226 | + |
| 227 | + closedir DP; |
| 228 | + |
| 229 | + unshift(@dirs, sort @subdirs) if $recurse_opt; |
| 230 | +} |
| 231 | + |
| 232 | +exit $exit_code; |
| 233 | + |
| 234 | +sub usage |
| 235 | +{ |
| 236 | + die <<EOT; |
| 237 | +Usage: rsyncsums [OPTIONS] [DIRS] |
| 238 | + |
| 239 | +Options: |
| 240 | + -r, --recurse Update checksums in subdirectories too. |
| 241 | + -l, --list List the checksums for each file (doesn't update). |
| 242 | + -c, --check Check if the checksums are right (doesn't update). |
| 243 | + -v, --verbose Mention what we're doing. Repeat for more info. |
| 244 | + -h, --help Display this help message. |
| 245 | +EOT |
| 246 | +} |
| 247 | diff --git a/xattrs.c b/xattrs.c |
| 248 | --- a/xattrs.c |
| 249 | +++ b/xattrs.c |
| 250 | @@ -34,6 +34,8 @@ extern int read_only; |
| 251 | extern int list_only; |
| 252 | extern int preserve_xattrs; |
| 253 | extern int checksum_seed; |
| 254 | +extern int checksum_len; |
| 255 | +extern int protocol_version; |
| 256 | |
| 257 | #define RSYNC_XAL_INITIAL 5 |
| 258 | #define RSYNC_XAL_LIST_INITIAL 100 |
| 259 | @@ -69,6 +71,10 @@ extern int checksum_seed; |
| 260 | #define XACC_ACL_ATTR RSYNC_PREFIX "%" XACC_ACL_SUFFIX |
| 261 | #define XDEF_ACL_SUFFIX "dacl" |
| 262 | #define XDEF_ACL_ATTR RSYNC_PREFIX "%" XDEF_ACL_SUFFIX |
| 263 | +#define MD4_SUFFIX "md4" |
| 264 | +#define MD4_ATTR RSYNC_PREFIX "%" MD4_SUFFIX |
| 265 | +#define MD5_SUFFIX "md5" |
| 266 | +#define MD5_ATTR RSYNC_PREFIX "%" MD5_SUFFIX |
| 267 | |
| 268 | typedef struct { |
| 269 | char *datum, *name; |
| 270 | @@ -241,7 +247,9 @@ static int rsync_xal_get(const char *fname, item_list *xalp) |
| 271 | || (am_root < 0 |
| 272 | && (strcmp(name+RPRE_LEN+1, XSTAT_SUFFIX) == 0 |
| 273 | || strcmp(name+RPRE_LEN+1, XACC_ACL_SUFFIX) == 0 |
| 274 | - || strcmp(name+RPRE_LEN+1, XDEF_ACL_SUFFIX) == 0))) |
| 275 | + || strcmp(name+RPRE_LEN+1, XDEF_ACL_SUFFIX) == 0 |
| 276 | + || strcmp(name+RPRE_LEN+1, MD4_SUFFIX) == 0 |
| 277 | + || strcmp(name+RPRE_LEN+1, MD5_SUFFIX) == 0))) |
| 278 | continue; |
| 279 | } |
| 280 | |
| 281 | @@ -947,6 +955,39 @@ int del_def_xattr_acl(const char *fname) |
| 282 | } |
| 283 | #endif |
| 284 | |
| 285 | +int get_sum_xattr(const char *fname, STRUCT_STAT *stp, char *sum) |
| 286 | +{ |
| 287 | + const char *mdattr = protocol_version >= 30 |
| 288 | + ? MD5_ATTR : MD4_ATTR; |
| 289 | + char buf[256]; |
| 290 | + uint32 file_length, mtime; |
| 291 | + int len; |
| 292 | + |
| 293 | + len = sys_lgetxattr(fname, mdattr, buf, sizeof buf); |
| 294 | + if (len < 0) { |
| 295 | + if (errno == ENOTSUP || errno == ENOATTR) |
| 296 | + return 0; |
| 297 | + rsyserr(FERROR_XFER, errno, "failed to read xattr %s for %s", |
| 298 | + mdattr, full_fname(fname)); |
| 299 | + return 0; |
| 300 | + } |
| 301 | + if (len != 4 + 4 + checksum_len) { |
| 302 | + rprintf(FERROR, "Corrupt %s xattr attached to %s -- skipping\n", |
| 303 | + mdattr, full_fname(fname)); |
| 304 | + return 0; |
| 305 | + } |
| 306 | + |
| 307 | + file_length = IVAL(buf, 0); /* 32-bit values -- trunctions are OK */ |
| 308 | + mtime = IVAL(buf, 4); |
| 309 | + |
| 310 | + if ((uint32)stp->st_size != file_length || (uint32)stp->st_mtime != mtime) |
| 311 | + return 0; |
| 312 | + |
| 313 | + memcpy(sum, buf + 8, checksum_len); |
| 314 | + |
| 315 | + return 1; |
| 316 | +} |
| 317 | + |
| 318 | int get_stat_xattr(const char *fname, int fd, STRUCT_STAT *fst, STRUCT_STAT *xst) |
| 319 | { |
| 320 | int mode, rdev_major, rdev_minor, uid, gid, len; |