Updated patches to work with the current trunk.

[rsync/rsync-patches.git] / checksum-xattr.diff

This patch is the start of storing/using checksum information from
extended attribute values.  The rsync code only reads the values
at the moment.  There is also a perl script that can create them.

To use this patch, run these commands for a successful build:

    patch -p1 <patches/checksum-xattr.diff
    ./configure                               (optional if already run)
    make

based-on: 24079e988fc31af4eba56cd2701fdc5a4154980d
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -1308,7 +1308,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 #endif
 
 	if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
-		file_checksum(thisname, tmp_sum, st.st_size);
+		if (!get_sum_xattr(thisname, &st, tmp_sum))
+			file_checksum(thisname, tmp_sum, st.st_size);
 		if (sender_keeps_checksum)
 			extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
 	}
diff --git a/generator.c b/generator.c
--- a/generator.c
+++ b/generator.c
@@ -531,7 +531,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
 	   of the file time to determine whether to sync */
 	if (always_checksum > 0 && S_ISREG(st->st_mode)) {
 		char sum[MAX_DIGEST_LEN];
-		file_checksum(fn, sum, st->st_size);
+		if (!get_sum_xattr(fn, st, sum))
+			file_checksum(fn, sum, st->st_size);
 		return memcmp(sum, F_SUM(file), checksum_len) == 0;
 	}
 
diff --git a/support/xsums b/support/xsums
new file mode 100755
--- /dev/null
+++ b/support/xsums
@@ -0,0 +1,204 @@
+#!/usr/bin/perl -w
+use strict;
+
+use Getopt::Long;
+use Cwd qw(abs_path cwd);
+use Digest::MD4;
+use Digest::MD5;
+use File::ExtAttr ':all';
+
+&Getopt::Long::Configure('bundling');
+&usage if !&GetOptions(
+    'recurse|r' => \( my $recurse_opt ),
+    'list|l' => \( my $list_opt ),
+    'check|c' => \( my $check_opt ),
+    'verbose|v+' => \( my $verbosity = 0 ),
+    'help|h' => \( my $help_opt ),
+);
+&usage if $help_opt;
+
+my $start_dir = cwd();
+
+my @dirs = @ARGV;
+@dirs = '.' unless @dirs;
+foreach (@dirs) {
+    $_ = abs_path($_);
+}
+
+$| = 1;
+
+my $exit_code = 0;
+
+my $md4 = Digest::MD4->new;
+my $md5 = Digest::MD5->new;
+
+while (@dirs) {
+    my $dir = shift @dirs;
+
+    if (!chdir($dir)) {
+	warn "Unable to chdir to $dir: $!\n";
+	next;
+    }
+    if (!opendir(DP, '.')) {
+	warn "Unable to opendir $dir: $!\n";
+	next;
+    }
+
+    my $reldir = $dir;
+    $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
+    print "$reldir ... " if $verbosity;
+
+    my @subdirs;
+    my $d_cnt = 0;
+    my $need_newline = $verbosity;
+    while (defined(my $fn = readdir(DP))) {
+	next if $fn =~ /^\.\.?$/ || -l $fn;
+	if (-d _) {
+	    push(@subdirs, "$dir/$fn");
+	    next;
+	}
+	next unless -f _;
+	$d_cnt++;
+
+	my($size,$mtime,$ctime) = (stat(_))[7,9,10];
+
+	my $xsum4 = getfattr($fn, 'rsync.%md4');
+	my $xsum5 = getfattr($fn, 'rsync.%md5');
+
+	my $sum_count = 0;
+	foreach ($xsum4, $xsum5) {
+	    if (defined $_) {
+		if (length($_) == 24) {
+		    my($sz,$mt,$sum) = unpack('V2a16', $_);
+		    if ($sz != ($size & 0xFFFFFFFF)
+		     || $mt != ($mtime & 0xFFFFFFFF)) {
+			$_ = undef;
+		    } else {
+			$_ = $sum;
+			$sum_count++;
+		    }
+		} else {
+		    $_ = undef;
+		}
+	    }
+	}
+
+	if ($list_opt) {
+	    if ($need_newline) {
+		print "\n";
+		$need_newline = 0;
+	    }
+	    if (defined $xsum4) {
+		print ' ', unpack('H32', $xsum4);
+	    } else {
+		print ' ' x (1 + 32);
+	    }
+	    if (defined $xsum5) {
+		print ' ', unpack('H32', $xsum5);
+	    } else {
+		print ' ' x (1 + 32);
+	    }
+	    print $verbosity ? ' ' : " $reldir/";
+	    print $fn, "\n";
+	    next;
+	}
+
+	if ($check_opt) {
+	    if (!$sum_count) {
+		if ($need_newline) {
+		    print "\n";
+		    $need_newline = 0;
+		}
+		print ' ' x (1 + 32 + 1 + 32) if $verbosity > 2;
+		print $verbosity ? ' ' : "$reldir/";
+		print $fn, " MISSING\n";
+		next;
+	    }
+	} else {
+	    next if $sum_count == 2;
+	    print 'UPDATING' if $need_newline && $verbosity == 1;
+	}
+
+	if ($need_newline && (!$check_opt || $verbosity > 1)) {
+	    print "\n";
+	    $need_newline = 0;
+	}
+
+	if (!open(IN, $fn)) {
+	    print STDERR "Unable to read $fn: $!\n";
+	    next;
+	}
+
+	my($sum4, $sum5);
+	while (1) {
+	    while (sysread(IN, $_, 64*1024)) {
+		$md4->add($_);
+		$md5->add($_);
+	    }
+	    $sum4 = $md4->digest;
+	    $sum5 = $md5->digest;
+	    print ' ', unpack('H32', $sum4), ' ', unpack('H32', $sum5) if $verbosity > 2;
+	    print " $fn" if $verbosity > 1;
+	    my($size2,$mtime2,$ctime2) = (stat(IN))[7,9,10];
+	    last if $size == $size2 && $mtime == $mtime2 && $ctime == $ctime2;
+	    $size = $size2;
+	    $mtime = $mtime2;
+	    $ctime = $ctime2;
+	    sysseek(IN, 0, 0);
+	    print " REREADING\n" if $verbosity > 1;
+	}
+
+	close IN;
+
+	if ($check_opt) {
+	    if ((!defined $xsum4 || $xsum4 eq $sum4) && (!defined $xsum5 || $xsum5 eq $sum5)) {
+		print " OK\n" if $verbosity > 1;
+		next;
+	    }
+	    if ($need_newline) {
+		print "\n";
+		$need_newline = 0;
+	    }
+	    if ($verbosity < 2) {
+		print $verbosity ? ' ' : "$reldir/";
+		print $fn;
+	    }
+	    print " FAILED\n";
+	    $exit_code = 1;
+	} else {
+	    print "\n" if $verbosity > 1;
+	    my $szmt = pack('V2', $size, $mtime); # 32-bits, may truncate
+	    setfattr($fn, 'rsync.%md4', $szmt.$sum4);
+	    setfattr($fn, 'rsync.%md5', $szmt.$sum5);
+	    #utime $mtime, $mtime, $fn; # Set mtime if it changes.
+	}
+    }
+
+    if ($need_newline) {
+	if ($d_cnt) {
+	    print "ok\n";
+	} else {
+	    print "empty\n";
+	}
+    }
+
+    closedir DP;
+
+    unshift(@dirs, sort @subdirs) if $recurse_opt;
+}
+
+exit $exit_code;
+
+sub usage
+{
+    die <<EOT;
+Usage: rsyncsums [OPTIONS] [DIRS]
+
+Options:
+ -r, --recurse     Update checksums in subdirectories too.
+ -l, --list        List the checksums for each file (doesn't update).
+ -c, --check       Check if the checksums are right (doesn't update).
+ -v, --verbose     Mention what we're doing.  Repeat for more info.
+ -h, --help        Display this help message.
+EOT
+}
diff --git a/xattrs.c b/xattrs.c
--- a/xattrs.c
+++ b/xattrs.c
@@ -34,6 +34,8 @@ extern int read_only;
 extern int list_only;
 extern int preserve_xattrs;
 extern int checksum_seed;
+extern int checksum_len;
+extern int protocol_version;
 
 #define RSYNC_XAL_INITIAL 5
 #define RSYNC_XAL_LIST_INITIAL 100
@@ -69,6 +71,10 @@ extern int checksum_seed;
 #define XACC_ACL_ATTR RSYNC_PREFIX "%" XACC_ACL_SUFFIX
 #define XDEF_ACL_SUFFIX "dacl"
 #define XDEF_ACL_ATTR RSYNC_PREFIX "%" XDEF_ACL_SUFFIX
+#define MD4_SUFFIX "md4"
+#define MD4_ATTR RSYNC_PREFIX "%" MD4_SUFFIX
+#define MD5_SUFFIX "md5"
+#define MD5_ATTR RSYNC_PREFIX "%" MD5_SUFFIX
 
 typedef struct {
 	char *datum, *name;
@@ -241,7 +247,9 @@ static int rsync_xal_get(const char *fname, item_list *xalp)
 			 || (am_root < 0
 			  && (strcmp(name+RPRE_LEN+1, XSTAT_SUFFIX) == 0
 			   || strcmp(name+RPRE_LEN+1, XACC_ACL_SUFFIX) == 0
-			   || strcmp(name+RPRE_LEN+1, XDEF_ACL_SUFFIX) == 0)))
+			   || strcmp(name+RPRE_LEN+1, XDEF_ACL_SUFFIX) == 0
+			   || strcmp(name+RPRE_LEN+1, MD4_SUFFIX) == 0
+			   || strcmp(name+RPRE_LEN+1, MD5_SUFFIX) == 0)))
 				continue;
 		}
 
@@ -947,6 +955,39 @@ int del_def_xattr_acl(const char *fname)
 }
 #endif
 
+int get_sum_xattr(const char *fname, STRUCT_STAT *stp, char *sum)
+{
+	const char *mdattr = protocol_version >= 30
+			   ? MD5_ATTR : MD4_ATTR;
+	char buf[256];
+	uint32 file_length, mtime;
+	int len;
+
+	len = sys_lgetxattr(fname, mdattr, buf, sizeof buf);
+	if (len < 0) {
+		if (errno == ENOTSUP || errno == ENOATTR)
+			return 0;
+		rsyserr(FERROR_XFER, errno, "failed to read xattr %s for %s",
+			mdattr, full_fname(fname));
+		return 0;
+	}
+	if (len != 4 + 4 + checksum_len) {
+		rprintf(FERROR, "Corrupt %s xattr attached to %s -- skipping\n",
+			mdattr, full_fname(fname));
+		return 0;
+	}
+
+	file_length = IVAL(buf, 0); /* 32-bit values -- trunctions are OK */
+	mtime = IVAL(buf, 4);
+
+	if ((uint32)stp->st_size != file_length || (uint32)stp->st_mtime != mtime)
+		return 0;
+
+	memcpy(sum, buf + 8, checksum_len);
+
+	return 1;
+}
+
 int get_stat_xattr(const char *fname, int fd, STRUCT_STAT *fst, STRUCT_STAT *xst)
 {
 	int mode, rdev_major, rdev_minor, uid, gid, len;