| 1 | #!/usr/bin/env perl |
| 2 | |
| 3 | # Continuous mirroring script around inotifywait and rsync, as suggested by |
| 4 | # Buck Huppmann. Supports local and remote pushing. |
| 5 | # EXPERIMENTAL! THERE IS ABSOLUTELY NO WARRANTY! |
| 6 | # -- Matt McCutchen <hashproduct@gmail.com> |
| 7 | # See: http://www.kepreon.com/~matt/utils/#continusync |
| 8 | |
| 9 | # Usage: |
| 10 | # continusync path/to/srcdir/ path/to/destdir/ |
| 11 | # continusync path/to/srcdir/ [user@]host:path/to/destdir/ |
| 12 | |
| 13 | # It seems to work, but it runs rsync once per event, which is ridiculous. |
| 14 | # TODO: Event batching!!! |
| 15 | # TODO: Do the recursive deletion in perl instead of calling rm(1). |
| 16 | |
| 17 | use warnings; |
| 18 | use strict; |
| 19 | |
| 20 | # Configuration. TODO: Add options for these. |
| 21 | |
| 22 | # Let the rsyncs we invoke piggyback on the main ssh connection. |
| 23 | # For this to work, you have to set a ControlPath in your ~/.ssh/config ; |
| 24 | # see the ssh_config(5) man page. |
| 25 | our @rshArgs = ('ssh', '-o', 'ControlMaster auto'); |
| 26 | |
| 27 | our $csPath = 'continusync'; |
| 28 | |
| 29 | # Don't put -r or --delete here. |
| 30 | # cp2 :) |
| 31 | our @rsyncArgs = ('rsync', '-lE', '--chmod=ugo=rwX', '-i'); |
| 32 | |
| 33 | use IPC::Open2; |
| 34 | use IO::Handle; |
| 35 | |
| 36 | # readFully(fh, length) -> data |
| 37 | sub readFully(*$) { |
| 38 | my ($fh, $bytesLeft) = @_; |
| 39 | my ($buf, $off, $rv) = ('', 0); |
| 40 | while ($bytesLeft > 0) { |
| 41 | $rv = sysread($fh, $buf, $bytesLeft, $off); |
| 42 | return undef if $rv == 0; # HMMM: May lose partial read |
| 43 | die "Read error" unless $rv > 0; |
| 44 | $bytesLeft -= $rv; |
| 45 | $off += $rv; |
| 46 | } |
| 47 | return $buf; |
| 48 | } |
| 49 | |
| 50 | # writeFully(fh, data) |
| 51 | sub writeFully(*$) { |
| 52 | my ($fh, $buf) = @_; |
| 53 | my ($bytesLeft, $off, $rv) = (length($buf), 0); |
| 54 | while ($bytesLeft > 0) { |
| 55 | $rv = syswrite($fh, $buf, $bytesLeft, $off); |
| 56 | die "Write error" unless $rv > 0; |
| 57 | $bytesLeft -= $rv; |
| 58 | $off += $rv; |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | # readMsg(fh) -> (type, body) |
| 63 | sub readMsg(*) { |
| 64 | my ($fh) = @_; |
| 65 | my $head = readFully($fh, 8); |
| 66 | return (undef, undef) unless defined($head); |
| 67 | my ($type, $bodyLen) = unpack('NN', $head); |
| 68 | return ($type, readFully($fh, $bodyLen)); |
| 69 | } |
| 70 | |
| 71 | # writeMsg(fh, type, body) |
| 72 | sub writeMsg(*$$) { |
| 73 | my ($fh, $type, $body) = @_; |
| 74 | writeFully($fh, pack('NN/a*', $type, $body)); |
| 75 | } |
| 76 | |
| 77 | # Message types |
| 78 | #sub MSG_EXIT { 0; } |
| 79 | sub MSG_REMOTE_PATH { 1; } |
| 80 | sub MSG_PERFORMED { 2; } |
| 81 | sub MSG_RENAME { 3; } |
| 82 | sub MSG_DELREC { 4; } |
| 83 | sub MSG_DELETED { 5; } |
| 84 | |
| 85 | sub doServer($) { |
| 86 | my ($dest) = @_; |
| 87 | |
| 88 | chdir($dest); |
| 89 | |
| 90 | my ($type, $body); |
| 91 | while (($type, $body) = readMsg(STDIN), defined($type)) { |
| 92 | if ($type == MSG_RENAME) { |
| 93 | my ($src, $dest) = unpack('N/a*N/a*', $body); |
| 94 | rename($src, $dest); |
| 95 | writeMsg(STDOUT, MSG_PERFORMED, ''); |
| 96 | } elsif ($type == MSG_DELREC) { |
| 97 | my $victim = $body; |
| 98 | my ($rmPid, $fromRm); |
| 99 | $rmPid = open($fromRm, '-|', 'rm', '-rf', '-v', $victim); |
| 100 | my $rmLine; |
| 101 | while (defined($rmLine = <$fromRm>)) { |
| 102 | chomp($rmLine); |
| 103 | if ($rmLine =~ /^[^`]*`(.*)'[^']*$/) { |
| 104 | writeMsg(STDOUT, MSG_DELETED, $1); |
| 105 | } |
| 106 | } |
| 107 | close($fromRm); |
| 108 | waitpid($rmPid, 0); |
| 109 | writeMsg(STDOUT, MSG_PERFORMED, ''); |
| 110 | } |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | # The stuff below applies only to the client. |
| 115 | |
| 116 | our ($src, $dest); |
| 117 | our $localDestFH; |
| 118 | |
| 119 | our ($fromServer, $toServer, $serverPid); |
| 120 | our ($fromInwt, $inwtPid); |
| 121 | |
| 122 | sub clientQuit() { |
| 123 | print "Caught a signal. Shutting down.\n"; |
| 124 | |
| 125 | #print STDOUT "serverPid is $serverPid\n"; |
| 126 | close($fromServer); |
| 127 | close($toServer); |
| 128 | waitpid($serverPid, 0); |
| 129 | |
| 130 | #print STDOUT "inwtPid is $inwtPid\n"; |
| 131 | kill(2, $inwtPid); |
| 132 | close($fromInwt); |
| 133 | waitpid($inwtPid, 0); |
| 134 | |
| 135 | exit(0); |
| 136 | } |
| 137 | |
| 138 | sub doRsync($$@) { |
| 139 | my ($isRecursive, $isDelete, @paths) = @_; |
| 140 | |
| 141 | my ($rsyncPid, $toRsync); |
| 142 | $rsyncPid = open($toRsync, '|-', @rsyncArgs, |
| 143 | ($isRecursive ? '-r' : '-d'), ($isDelete ? '--del' : ()), |
| 144 | '--no-implied-dirs', '-t', '--from0', '--files-from=-', '.', $dest); |
| 145 | foreach my $p (@paths) { |
| 146 | print $toRsync $p, "\0"; |
| 147 | } |
| 148 | close($toRsync); |
| 149 | waitpid($rsyncPid, 0); |
| 150 | } |
| 151 | |
| 152 | sub doRename($$) { |
| 153 | my ($src, $dest) = @_; |
| 154 | writeMsg($toServer, MSG_RENAME, pack('N/a*N/a*', $src, $dest)); |
| 155 | readMsg($fromServer); # MSG_PERFORMED |
| 156 | print "*movefrom $src\n", |
| 157 | "*moveto $dest\n"; |
| 158 | } |
| 159 | |
| 160 | sub doDelete($) { |
| 161 | my ($path) = @_; |
| 162 | writeMsg($toServer, MSG_DELREC, $path); |
| 163 | my ($type, $body); |
| 164 | while (($type, $body) = readMsg($fromServer), $type == MSG_DELETED) { |
| 165 | print "*deleting $body\n"; |
| 166 | } |
| 167 | # Also reads the final MSG_PERFORMED. |
| 168 | } |
| 169 | |
| 170 | # move_self so we can reliably detect moves out |
| 171 | our @interestingEvents = ('modify', 'attrib', 'move', 'move_self', 'create', 'delete'); |
| 172 | |
| 173 | sub doClient($$) { |
| 174 | ($src, $dest) = @_; |
| 175 | |
| 176 | print "Continusync starting up.\n", |
| 177 | "This software is EXPERIMENTAL. There is ABSOLUTELY NO WARRANTY.\n"; |
| 178 | |
| 179 | # Get a server process. |
| 180 | # Echoes of rsync... |
| 181 | if ($dest =~ /^([^:]*):(.*)$/) { |
| 182 | # Invoke over remote shell |
| 183 | my ($uhost, $rdest) = ($1, $2); |
| 184 | $serverPid = open2($fromServer, $toServer, @rshArgs, $uhost, $csPath, '--server'); |
| 185 | # Pass path on stdin to stop the shell from messing with it. |
| 186 | # Echoes of rsync daemon protocol... |
| 187 | writeMsg($toServer, MSG_REMOTE_PATH, $rdest); |
| 188 | } else { |
| 189 | # Fork locally |
| 190 | my ($fromClient, $toClient); |
| 191 | pipe($fromServer, $toClient); |
| 192 | pipe($fromClient, $toServer); |
| 193 | $serverPid = fork(); |
| 194 | if ($serverPid == 0) { |
| 195 | # Child server |
| 196 | close($fromServer); |
| 197 | close($toServer); |
| 198 | open(STDIN, "<&", $fromClient); |
| 199 | open(STDOUT, ">&", $toClient); |
| 200 | doServer($dest); |
| 201 | exit(0); |
| 202 | } |
| 203 | # Parent client |
| 204 | close($fromClient); |
| 205 | close($toClient); |
| 206 | # Get a dest path that we can pass to rsync even after we chdir into the source. |
| 207 | { |
| 208 | local $^F = 100000; |
| 209 | open($localDestFH, '<', $dest); |
| 210 | } |
| 211 | $dest = "/proc/self/fd/" . fileno($localDestFH); |
| 212 | } |
| 213 | |
| 214 | chdir($src); |
| 215 | |
| 216 | # Get inotifywait. |
| 217 | $inwtPid = open($fromInwt, '-|'); |
| 218 | if ($inwtPid == 0) { |
| 219 | # Parent wants all our output on the single filehandle $fromInwt. |
| 220 | open(STDERR, ">&", STDOUT); |
| 221 | my @args = ('inotifywait', '-r', '-m', '--format', "%e\n%w\n%f", map(('-e', $_), @interestingEvents), '.'); |
| 222 | exec(@args); |
| 223 | } |
| 224 | |
| 225 | <$fromInwt>; # `Setting up watches' |
| 226 | <$fromInwt>; # `Watches established' |
| 227 | $SIG{INT} = \&clientQuit; |
| 228 | print "Continuously mirroring. Give me a SIGINT when you want me to quit.\n"; |
| 229 | |
| 230 | # Now we can do the initial copy without danger of losing events. |
| 231 | doRsync(1, 1, '.'); |
| 232 | |
| 233 | # Consecutive MOVED_FROM and MOVED_TO events constitute an internal |
| 234 | # move. A move-out followed by a move-in gives an intervening |
| 235 | # MOVED_SELF, so we aren't fooled. |
| 236 | my $movedFrom = undef; |
| 237 | |
| 238 | for (;;) { |
| 239 | my ($e, $w, $f); |
| 240 | chomp($e = <$fromInwt>); |
| 241 | chomp($w = <$fromInwt>); |
| 242 | chomp($f = <$fromInwt>); |
| 243 | my $path = $w . $f; |
| 244 | $path =~ s,^\./(.),$1,; # Remove initial ./ if it isn't all |
| 245 | my $isDir = ($e =~ s/,ISDIR$//); |
| 246 | #print "Got event: ($e,$isDir,$w,$f)\n"; |
| 247 | |
| 248 | if (defined($movedFrom)) { |
| 249 | if ($e eq 'MOVED_TO') { |
| 250 | # Complete the move. |
| 251 | doRename($movedFrom, $path); |
| 252 | next; |
| 253 | } else { |
| 254 | # Moved out. |
| 255 | doDelete($movedFrom); |
| 256 | } |
| 257 | $movedFrom = undef; |
| 258 | } |
| 259 | |
| 260 | if ($e eq 'MODIFY') { |
| 261 | doRsync(0, 0, $path); |
| 262 | } elsif ($e eq 'ATTRIB') { |
| 263 | doRsync(0, 0, $path); |
| 264 | } elsif ($e eq 'MOVED_FROM') { |
| 265 | $movedFrom = $path; |
| 266 | } elsif ($e eq 'MOVED_TO') { |
| 267 | # Moved in. |
| 268 | # Must be recursive in case it was an entire directory. |
| 269 | doRsync(1, 0, $path); |
| 270 | } elsif ($e eq 'CREATE') { |
| 271 | doRsync(0, 0, $path); |
| 272 | } elsif ($e eq 'DELETE') { |
| 273 | doDelete($path); |
| 274 | } |
| 275 | } |
| 276 | # not reached |
| 277 | } |
| 278 | |
| 279 | if ($ARGV[0] eq '--server') { |
| 280 | #STDOUT->autoflush(1); |
| 281 | my ($type, $dest) = readMsg(STDIN); |
| 282 | doServer($dest); |
| 283 | } else { |
| 284 | doClient($ARGV[0], $ARGV[1]); |
| 285 | } |
| 286 | |