This patch is the start of storing/using checksum information from extended attribute values. The rsync code only reads the values at the moment. There is also a perl script that can create them. To use this patch, run these commands for a successful build: patch -p1 0 && S_ISREG(st->st_mode)) { char sum[MAX_DIGEST_LEN]; - file_checksum(fn, sum, st->st_size); + if (!get_sum_xattr(fn, st, sum)) + file_checksum(fn, sum, st->st_size); return memcmp(sum, F_SUM(file), checksum_len) == 0; } diff --git a/support/xsums b/support/xsums new file mode 100755 --- /dev/null +++ b/support/xsums @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w +use strict; + +use Getopt::Long; +use Cwd qw(abs_path cwd); +use Digest::MD4; +use Digest::MD5; +use File::ExtAttr ':all'; + +&Getopt::Long::Configure('bundling'); +&usage if !&GetOptions( + 'recurse|r' => \( my $recurse_opt ), + 'list|l' => \( my $list_opt ), + 'check|c' => \( my $check_opt ), + 'verbose|v+' => \( my $verbosity = 0 ), + 'help|h' => \( my $help_opt ), +); +&usage if $help_opt; + +my $start_dir = cwd(); + +my @dirs = @ARGV; +@dirs = '.' unless @dirs; +foreach (@dirs) { + $_ = abs_path($_); +} + +$| = 1; + +my $exit_code = 0; + +my $md4 = Digest::MD4->new; +my $md5 = Digest::MD5->new; + +while (@dirs) { + my $dir = shift @dirs; + + if (!chdir($dir)) { + warn "Unable to chdir to $dir: $!\n"; + next; + } + if (!opendir(DP, '.')) { + warn "Unable to opendir $dir: $!\n"; + next; + } + + my $reldir = $dir; + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo; + print "$reldir ... " if $verbosity; + + my @subdirs; + my $d_cnt = 0; + my $need_newline = $verbosity; + while (defined(my $fn = readdir(DP))) { + next if $fn =~ /^\.\.?$/ || -l $fn; + if (-d _) { + push(@subdirs, "$dir/$fn"); + next; + } + next unless -f _; + $d_cnt++; + + my($size,$mtime,$ctime) = (stat(_))[7,9,10]; + + my $xsum4 = getfattr($fn, 'rsync.%md4'); + my $xsum5 = getfattr($fn, 'rsync.%md5'); + + my $sum_count = 0; + foreach ($xsum4, $xsum5) { + if (defined $_) { + if (length($_) == 24) { + my($sz,$mt,$sum) = unpack('V2a16', $_); + if ($sz != ($size & 0xFFFFFFFF) + || $mt != ($mtime & 0xFFFFFFFF)) { + $_ = undef; + } else { + $_ = $sum; + $sum_count++; + } + } else { + $_ = undef; + } + } + } + + if ($list_opt) { + if ($need_newline) { + print "\n"; + $need_newline = 0; + } + if (defined $xsum4) { + print ' ', unpack('H32', $xsum4); + } else { + print ' ' x (1 + 32); + } + if (defined $xsum5) { + print ' ', unpack('H32', $xsum5); + } else { + print ' ' x (1 + 32); + } + print $verbosity ? ' ' : " $reldir/"; + print $fn, "\n"; + next; + } + + if ($check_opt) { + if (!$sum_count) { + if ($need_newline) { + print "\n"; + $need_newline = 0; + } + print ' ' x (1 + 32 + 1 + 32) if $verbosity > 2; + print $verbosity ? ' ' : "$reldir/"; + print $fn, " MISSING\n"; + next; + } + } else { + next if $sum_count == 2; + print 'UPDATING' if $need_newline && $verbosity == 1; + } + + if ($need_newline && (!$check_opt || $verbosity > 1)) { + print "\n"; + $need_newline = 0; + } + + if (!open(IN, $fn)) { + print STDERR "Unable to read $fn: $!\n"; + next; + } + + my($sum4, $sum5); + while (1) { + while (sysread(IN, $_, 64*1024)) { + $md4->add($_); + $md5->add($_); + } + $sum4 = $md4->digest; + $sum5 = $md5->digest; + print ' ', unpack('H32', $sum4), ' ', unpack('H32', $sum5) if $verbosity > 2; + print " $fn" if $verbosity > 1; + my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10]; + last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2; + $size = $size2; + $mtime = $mtime2; + $ctime = $ctime2; + sysseek(IN, 0, 0); + print " REREADING\n" if $verbosity > 1; + } + + close IN; + + if ($check_opt) { + if ((!defined $xsum4 || $xsum4 eq $sum4) && (!defined $xsum5 || $xsum5 eq $sum5)) { + print " OK\n" if $verbosity > 1; + next; + } + if ($need_newline) { + print "\n"; + $need_newline = 0; + } + if ($verbosity < 2) { + print $verbosity ? ' ' : "$reldir/"; + print $fn; + } + print " FAILED\n"; + $exit_code = 1; + } else { + print "\n" if $verbosity > 1; + my $szmt = pack('V2', $size, $mtime); # 32-bits, may truncate + setfattr($fn, 'rsync.%md4', $szmt.$sum4); + setfattr($fn, 'rsync.%md5', $szmt.$sum5); + #utime $mtime, $mtime, $fn; # Set mtime if it changes. + } + } + + if ($need_newline) { + if ($d_cnt) { + print "ok\n"; + } else { + print "empty\n"; + } + } + + closedir DP; + + unshift(@dirs, sort @subdirs) if $recurse_opt; +} + +exit $exit_code; + +sub usage +{ + die <= 30 + ? MD5_ATTR : MD4_ATTR; + char buf[256]; + uint32 file_length, mtime; + int len; + + len = sys_lgetxattr(fname, mdattr, buf, sizeof buf); + if (len < 0) { + if (errno == ENOTSUP || errno == ENOATTR) + return 0; + rsyserr(FERROR_XFER, errno, "failed to read xattr %s for %s", + mdattr, full_fname(fname)); + return 0; + } + if (len != 4 + 4 + checksum_len) { + rprintf(FERROR, "Corrupt %s xattr attached to %s -- skipping\n", + mdattr, full_fname(fname)); + return 0; + } + + file_length = IVAL(buf, 0); /* 32-bit values -- trunctions are OK */ + mtime = IVAL(buf, 4); + + if ((uint32)stp->st_size != file_length || (uint32)stp->st_mtime != mtime) + return 0; + + memcpy(sum, buf + 8, checksum_len); + + return 1; +} + int get_stat_xattr(const char *fname, int fd, STRUCT_STAT *fst, STRUCT_STAT *xst) { int mode, rdev_major, rdev_minor, uid, gid, len;