sanoid/syncoid

#!/usr/bin/perl

# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved 
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.

my $version = '1.0.16';

use strict;
use Data::Dumper;
use Time::Local;
use Sys::Hostname;

my %args = getargs(@ARGV);

if ($args{'version'}) {
	print "Syncoid version: $version\n";
	exit 0;
}

my $rawsourcefs = $args{'source'};
my $rawtargetfs = $args{'target'};
my $debug = $args{'debug'};

my $zfscmd = '/sbin/zfs';
my $sshcmd = '/usr/bin/ssh';
my $pscmd = '/bin/ps';
my $sshcipher = '-c arcfour';
my $pvcmd = '/usr/bin/pv';
my $mbuffercmd = '/usr/bin/mbuffer';
my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null';
# currently using ls to check for file existence because we aren't depending on perl
# being present on remote machines.
my $lscmd = '/bin/ls';

# figure out if source and/or target are remote.
my ($sourcehost,$sourcefs) = getssh($rawsourcefs);
my ($targethost,$targetfs) = getssh($rawtargetfs);

# make sure target is not currently in receive.
if (iszfsbusy($targethost,$targetfs)) {
	die "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
}

# figure out whether compression, mbuffering, pv
# are available on source, target, local machines.
# warn user of anything missing, then continue with sync.
my %avail = checkcommands();
$sshcmd = "$sshcmd $sshcipher";
# does the target filesystem exist yet?
my $targetexists = targetexists($targethost,$targetfs);

# build hashes of the snaps on the source and target filesystems.
my %snaps;
%snaps = getsnaps(\%snaps,'source',$sourcehost,$sourcefs);
if ($targetexists) { %snaps = getsnaps(\%snaps,'target',$targethost,$targetfs); }

# create a new syncoid snapshot on the source filesystem.
my $newsyncsnap = newsyncsnap($sourcehost,$sourcefs);

# there is currently (2014-09-01) a bug in ZFS on Linux
# that causes readonly to always show on if it's EVER
# been turned on... even when it's off... unless and
# until the filesystem is zfs umounted and zfs remounted.
# we're going to do the right thing anyway.
my $originaltargetreadonly;

# sync 'em up.
if (! $targetexists) {
	# do an initial sync from the oldest source snapshot
	# THEN do an -I to the newest
	my $oldestsnap = getoldestsnapshot(\%snaps);
	my $sendcmd = "$zfscmd send $sourcefs\@$oldestsnap";
	my $recvcmd = "$zfscmd receive -F $targetfs";
	my $pvsize = getsendsize("$sourcefs\@$oldestsnap");
	my $disp_pvsize = readablebytes($pvsize);
	if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; }
	my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize);
	print "Sending oldest full snapshot $oldestsnap (~ $disp_pvsize) to new target filesystem:\n";
	if ($debug) { print "DEBUG: $synccmd\n"; }

	# make sure target is (still) not currently in receive.
	if (iszfsbusy($targethost,$targetfs)) {
		die "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
	}
	system($synccmd);

	# now do an -I to the new sync snapshot, assuming there were any snapshots
	# other than the new sync snapshot to begin with, of course
	if ($oldestsnap ne $newsyncsnap) {

		# get current readonly status of target, then set it to on during sync
		$originaltargetreadonly = getzfsvalue($targethost,$targetfs,'readonly');
		setzfsvalue($targethost,$targetfs,'readonly','on');

		$sendcmd = "$zfscmd send -I $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap";
		$pvsize = getsendsize("$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap");
		$disp_pvsize = readablebytes($pvsize);
		if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
		$synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize);

		# make sure target is (still) not currently in receive.
		if (iszfsbusy($targethost,$targetfs)) {
			die "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
		}

		print "Updating new target filesystem with incremental $oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n";
		if ($debug) { print "DEBUG: $synccmd\n"; }
		system($synccmd);

		# restore original readonly value to target after sync complete
		setzfsvalue($targethost,$targetfs,'readonly',$originaltargetreadonly);			
	}
} else {
	# find most recent matching snapshot and do an -I
	# to the new snapshot

	# get current readonly status of target, then set it to on during sync
	$originaltargetreadonly = getzfsvalue($targethost,$targetfs,'readonly');
	setzfsvalue($targethost,$targetfs,'readonly','on');

	my $matchingsnap = getmatchingsnapshot(\%snaps);

	# make sure target is (still) not currently in receive.
	if (iszfsbusy($targethost,$targetfs)) {
		die "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
	}

	# rollback target to matchingsnap
	if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
	if ($targethost ne '') {
		system ("$sshcmd $targethost $zfscmd rollback -R $targetfs\@$matchingsnap");
	} else {
		system ("$zfscmd rollback -R $targetfs\@$matchingsnap");
	}

	my $sendcmd = "$zfscmd send -I $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap";
	my $recvcmd = "$zfscmd receive $targetfs";
	my $pvsize = getsendsize("$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap");
	my $disp_pvsize = readablebytes($pvsize);
	if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
	my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize);

	print "Sending incremental $matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n";
	if ($debug) { print "DEBUG: $synccmd\n"; }
	system("$synccmd");

	# restore original readonly value to target after sync complete
	setzfsvalue($targethost,$targetfs,'readonly',$originaltargetreadonly);			
}

# prune obsolete sync snaps on source and target.
pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,keys %{ $snaps{'source'}});
pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,keys %{ $snaps{'target'}});

# debug: print contents of %snaps to stdout
#dumphash(\%snaps);

exit;

##############################################################################
##############################################################################
##############################################################################
##############################################################################


sub getargs {
	my @args = @_;
	my %args;

	my %novaluearg;
	my %validarg;
	push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit');
	foreach my $item (@validargs) { $validarg{$item} = 1; }
	push my @novalueargs, ('debug','nocommandchecks','version','monitor-version');
	foreach my $item (@novalueargs) { $novaluearg{$item} = 1; }

	while (my $rawarg = shift(@args)) {
		my $arg = $rawarg;
		my $argvalue;
		if ($rawarg =~ /=/) {
			# user specified the value for a CLI argument with =
			# instead of with blank space. separate appropriately.
			$argvalue = $arg;
			$arg =~ s/=.*$//;
			$argvalue =~ s/^.*=//;
		}
		if ($rawarg =~ /^--/) {
			# doubledash arg
			$arg =~ s/^--//;
			if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
			if ($novaluearg{$arg}) {
				$args{$arg} = 1;
			} else {
				# if this CLI arg takes a user-specified value and
				# we don't already have it, then the user must have
				# specified with a space, so pull in the next value
				# from the array as this value rather than as the 
				# next argument.
				if ($argvalue eq '') { $argvalue = shift(@args); }
				$args{$arg} = $argvalue;
			}
		} elsif ($arg =~ /^-/) {
			# singledash arg
			$arg =~ s/^-//;
			if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
			if ($novaluearg{$arg}) {
				$args{$arg} = 1;
			} else {
				# if this CLI arg takes a user-specified value and
				# we don't already have it, then the user must have
				# specified with a space, so pull in the next value
				# from the array as this value rather than as the 
				# next argument.
				if ($argvalue eq '') { $argvalue = shift(@args); }
				$args{$arg} = $argvalue;
			}
		} else {
			# bare arg
			if (defined $args{'source'}) {
				if (! defined $args{'target'}) {
					$args{'target'} = $arg;
				} else {
					die "ERROR: don't know what to do with third bare argument $rawarg.\n";
				}
			} else {
				$args{'source'} = $arg;
			}
		}
	}

	if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; }
	if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; }

	if ($args{'compress'} eq 'gzip') {
		$args{'rawcompresscmd'} = '/bin/gzip';
		$args{'compressargs'} = '-3';
		$args{'rawdecompresscmd'} = '/bin/zcat';
		$args{'decompressargs'} = '';
	} elsif ( ($args{'compress'} eq 'lzo') || ! (defined $args{'compress'}) ) {
		$args{'rawcompresscmd'} = '/usr/bin/lzop';
		$args{'compressargs'} = '';
		$args{'rawdecompresscmd'} = '/usr/bin/lzop';
		$args{'decompressargs'} = '-dfc';
	} else {
		$args{'rawcompresscmd'} = '';
		$args{'compressargs'} = '';
		$args{'rawdecompresscmd'} = '';
		$args{'decompressargs'} = '';
	}
	$args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}";
	$args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}";

	return %args;
}

sub checkcommands {
	# make sure compression, mbuffer, and pv are available on
	# source, target, and local hosts as appropriate.

	my %avail;
	my $sourcessh;
	my $targetssh;

	# if --nocommandchecks then assume everything's available and return
	if ($args{'nocommandchecks'}) { 
		if ($debug) { print "DEBUG: not checking for command availability due to --nocommandchecks switch.\n"; }
		$avail{'compress'} = 1;
		$avail{'localpv'} = 1;
		$avail{'localmbuffer'} = 1;
		$avail{'sourcembuffer'} = 1;
		$avail{'targetmbuffer'} = 1;
		return %avail; 
	}

	if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; }
	if ($targethost ne '') { $targetssh = "$sshcmd $targethost"; }

	# if raw compress command is null, we must have specified no compression. otherwise, 
	# make sure that compression is available everywhere we need it
	if ($args{'rawcompresscmd'} eq '') {
		$avail{'sourcecompress'} = 0;
		$avail{'sourcecompress'} = 0;
		$avail{'localcompress'} = 0;
		if ($args{'compress'} eq 'none' ||
		    $args{'compress'} eq 'no' ||
		    $args{'compress'} eq '0') {
			if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; }
		} else {
			print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n";
		}
	} else {
		if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; }
		$avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
		if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; }
		$avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
		if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; }
		$avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`;
	}

	my ($s,$t);
	if ($sourcehost eq '') { 
		$s = '[local machine]'
	} else {
		$s = $sourcehost;
		$s =~ s/^\S*\@//;
		$s = "ssh:$s";
	}
	if ($targethost eq '') { 
		$t = '[local machine]'
	} else {
		$t = $targethost;
		$t =~ s/^\S*\@//;
		$t = "ssh:$t";
	}

	if ($avail{'sourcecompress'} eq '') { 
		if ($args{'rawcompresscmd'} ne '') {
			print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n"; 
		}
		$avail{'compress'} = 0;
	}
	if ($avail{'targetcompress'} eq '') {
		if ($args{'rawcompresscmd'} ne '') {
			print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n"; 
		}
		$avail{'compress'} = 0;
	}
	if ($avail{'targetcompress'} ne '' && $avail{'sourcecompress'} ne '') {
		# compression available - unless source and target are both remote, which we'll check
		# for in the next block and respond to accordingly.
		$avail{'compress'} = 1;
	}

	# corner case - if source AND target are BOTH remote, we have to check for local compress too
	if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') { 
		if ($args{'rawcompresscmd'} ne '') {
			print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n"; 
		}
		$avail{'compress'} = 0;
	}

	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; }
	$avail{'sourcembuffer'} = `$sourcessh $lscmd $mbuffercmd 2>/dev/null`;
	if ($avail{'sourcembuffer'} eq '') {
		print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n";
		$avail{'sourcembuffer'} = 0;
	} else {
		$avail{'sourcembuffer'} = 1;
	}

	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; }
	$avail{'targetmbuffer'} = `$targetssh $lscmd $mbuffercmd 2>/dev/null`;
	if ($avail{'targetmbuffer'} eq '') {
		print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n";
		$avail{'targetmbuffer'} = 0;
	} else {
		$avail{'targetmbuffer'} = 1;
	}

	# if we're doing remote source AND remote target, check for local mbuffer as well
	if ($sourcehost ne '' && $targethost ne '') {
		if ($debug) { print "DEBUG: checking availability of $mbuffercmd on local machine...\n"; }
		$avail{'localmbuffer'} = `$lscmd $mbuffercmd 2>/dev/null`;
		if ($avail{'localmbuffer'} eq '') {
			$avail{'localmbuffer'} = 0;
			print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n";
		}
	}

	if ($debug) { print "DEBUG: checking availability of $pvcmd on local machine...\n"; }
	$avail{'localpv'} = `$lscmd $pvcmd 2>/dev/null`;
	if ($avail{'localpv'} eq '') {
		print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n";
		$avail{'localpv'} = 0;
	} else {
		$avail{'localpv'} = 1;
	}
	
	return %avail;
}

sub iszfsbusy {
	my ($rhost,$fs) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	if ($debug) { print "DEBUG: checking to see if $fs on $rhost is already in zfs receive using $rhost $pscmd axo args= ...\n"; }

	open PL, "$rhost $pscmd axo args= |";
	my @processes = <PL>;
	close PL;

	foreach my $process (@processes) {
		# if ($debug) { print "DEBUG: checking process $process...\n"; }
		if ($process =~ /zfs receive.*$fs/) {
			# there's already a zfs receive process for our target filesystem - return true
			if ($debug) { print "DEBUG: process $process matches target $fs!\n"; }
			return 1;
		}
	}

	# no zfs receive processes for our target filesystem found - return false
	return 0;
}

sub setzfsvalue {
	my ($rhost,$fs,$property,$value) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	if ($debug) { print "DEBUG: setting $property to $value on $fs...\n"; }
	system("$rhost $zfscmd set $property=$value $fs");
	return;
}

sub getzfsvalue {
	my ($rhost,$fs,$property) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	if ($debug) { print "DEBUG: getting current value of $property on $fs...\n"; }
	open FH, "$rhost $zfscmd get -H $property $fs |";
	my $value = <FH>;
	close FH;
	my @values = split(/\s/,$value);
	$value = $values[2];
	return $value;
}

sub readablebytes {
	my $bytes = shift;
	my $disp;

	if ($bytes > 1024*1024*1024) {
		$disp = sprintf("%.1f",$bytes/1024/1024/1024) . ' GB';
	} elsif ($bytes > 1024*1024) {
		$disp = sprintf("%.1f",$bytes/1024/1024) . ' MB';
	} else {
		$disp = sprintf("%d",$bytes/1024) . ' KB';
	}
	return $disp;
}

sub getoldestsnapshot {
	my $snaps = shift;
	foreach my $snap ( sort { $snaps{'source'}{$a}{'ctime'}<=>$snaps{'source'}{$b}{'ctime'} } keys %{ $snaps{'source'} }) {
		# return on first snap found - it's the oldest
		return $snap;
	}
	# must not have had any snapshots on source - luckily, we already made one, amirite?
	return $newsyncsnap;
}

sub buildsynccmd {
	my ($sendcmd,$recvcmd,$pvsize) = @_;
	# here's where it gets fun: figuring out when to compress and decompress.
	# to make this work for all possible combinations, you may have to decompress
	# AND recompress across the pipe viewer. FUN.
	my $synccmd;

	if ($sourcehost eq '' && $targethost eq '') {
		# both sides local. don't compress. do mbuffer, once, on the source side.
		# $synccmd = "$sendcmd | $mbuffercmd | $pvcmd | $recvcmd";
		$synccmd = "$sendcmd |";
		# avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here
		my $bwlimit;
		if ($args{'source-bwlimit'} eq '') { 
			$bwlimit = $args{'target-bwlimit'}; 
		} else { 
			$bwlimit = $args{'source-bwlimit'}; 
		}
		if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $bwlimit $mbufferoptions |"; }
		if ($avail{'localpv'}) { $synccmd .= " $pvcmd -s $pvsize |"; }
		$synccmd .= " $recvcmd";
	} elsif ($sourcehost eq '') {
		# local source, remote target.
		#$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
		$synccmd = "$sendcmd |";
		if ($avail{'localpv'}) { $synccmd .= " $pvcmd -s $pvsize |"; }
		if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; }
		if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
		$synccmd .= " $sshcmd $targethost '";
		if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
		if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; }
		$synccmd .= " $recvcmd'";
	} elsif ($targethost eq '') {
		# remote source, local target.
		#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd";
		$synccmd = "$sshcmd $sourcehost '$sendcmd";
		if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
		if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
		$synccmd .= "' | ";
		if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
		if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
		if ($avail{'localpv'}) { $synccmd .= "$pvcmd -s $pvsize | "; }
		$synccmd .= $recvcmd;
	} else {
		#remote source, remote target... weird, but whatever, I'm not here to judge you.
		#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
		$synccmd = "$sshcmd $sourcehost '$sendcmd";
		if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
		if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
		$synccmd .= "' | ";
		if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
		if ($avail{'localpv'}) { $synccmd .= "$pvcmd -s $pvsize | "; }
		if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; }
		if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; }
		$synccmd .= "$sshcmd $targethost '";
		if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
		if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
		$synccmd .= "$recvcmd'";
	}
	return $synccmd;
}

sub pruneoldsyncsnaps {
	my ($rhost,$fs,$newsyncsnap,@snaps) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	my $hostid = hostname();

	my @prunesnaps;

	# only prune snaps beginning with syncoid and our own hostname
	foreach my $snap(@snaps) { 
		if ($snap =~ /^syncoid_$hostid/) {
			# no matter what, we categorically refuse to
			# prune the new sync snap we created for this run
			if ($snap ne $newsyncsnap) {
				push (@prunesnaps,$snap);
			}
		}
	}

	# concatenate pruning commands to ten per line, to cut down
	# auth times for any remote hosts that must be operated via SSH
	my $counter;
	my $maxsnapspercmd = 10;
	my $prunecmd;
	foreach my $snap(@prunesnaps) {
		$counter ++;
		$prunecmd .= "$zfscmd destroy $fs\@$snap; ";
		if ($counter > $maxsnapspercmd) {
			$prunecmd =~ s/\; $//;
			if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; }
			if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; }
			if ($debug) { print "DEBUG: $rhost $prunecmd\n"; }
			system("$rhost $prunecmd");
			$prunecmd = '';
			$counter = 0;
		}
	}
	# if we still have some prune commands stacked up after finishing
	# the loop, commit 'em now
	if ($counter) { 
		$prunecmd =~ s/\; $//; 
		if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; }
		if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; }
		if ($debug) { print "DEBUG: $rhost $prunecmd\n"; }
		system("$rhost $prunecmd"); 
	}
	return;
}

sub getmatchingsnapshot {
	my $snaps = shift;
	foreach my $snap ( sort { $snaps{'source'}{$b}{'ctime'}<=>$snaps{'source'}{$a}{'ctime'} } keys %{ $snaps{'source'} }) {
		if ($snaps{'source'}{$snap}{'ctime'} == $snaps{'target'}{$snap}{'ctime'}) {
			return $snap;
		}
	}
	print "UNEXPECTED ERROR: target exists but has no matching snapshots!\n";
	exit 256;
}

sub newsyncsnap {
	my ($rhost,$fs) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	my $hostid = hostname();
	my %date = getdate();
	my $snapname = "syncoid\_$hostid\_$date{'stamp'}";
	my $snapcmd = "$rhost $zfscmd snapshot $fs\@$snapname\n";
	system($snapcmd);
	return $snapname;
}

sub targetexists {
	my ($rhost,$fs) = @_;
	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
	if ($debug) { print "DEBUG: checking to see if target filesystem exists...\n"; }
	open FH, "$rhost $zfscmd get -H name $fs 2>&1 |";
	$targetexists = <FH>;
	close FH;
	my $exit = $?;
	$targetexists = ( $targetexists =~ /^$fs/ && $exit == 0 );
	return $targetexists;
}

sub getssh {
	my $fs = shift;

	my $rhost;
	if ($fs =~ /\@/) {
		$rhost = $fs;
		$fs =~ s/^\S*\@\S*://;
		$rhost =~ s/:$fs$//;
	}
	return ($rhost,$fs);
}

sub dumphash() {
	my $hash = shift;
	$Data::Dumper::Sortkeys = 1;
	print Dumper($hash);
}

sub getsnaps() {
	my ($snaps,$type,$rhost,$fs) = @_;

	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }

	my $getsnapcmd = "$rhost $zfscmd get -Hpd 1 creation $fs |";
	if ($debug) { print "DEBUG: getting list of snapshots on $fs...\n"; }
	open FH, $getsnapcmd;
	my @rawsnaps = <FH>;
	close FH;

	foreach my $line (@rawsnaps) {
		# only import snaps from the specified filesystem
		if ($line =~ /$fs\@/) {
			chomp $line;
			my $ctime = $line;
			$ctime =~ s/^.*creation\s*//;
			$ctime =~ s/\s*-$//;
			my $snap = $line;
			$snap =~ s/\s*creation.*$//;
			$snap =~ s/^\S*\@//;
			$snaps{$type}{$snap}{'ctime'}=$ctime;
		}
	}
	return %snaps;
}


sub getsendsize { 
	my ($snap1,$snap2) = @_;

	my $snaps;
	if ($snap2 ne '') {
		$snaps = "-I $snap1 $snap2";
	} else {
		$snaps = "$snap1";
	}

	my $sourcessh;
	my $quote;
	if ($sourcehost ne '') {
		$sourcessh = "$sshcmd $sourcehost";
		$quote = '"';
	}

	if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost...\n"; }
	open FH, "$sourcessh $zfscmd send -nP $snaps 2>&1 |";
	my @rawsize = <FH>;
	close FH;
	my $exit = $?;

	# process sendsize: last line of multi-line output is 
	# size of proposed xfer in bytes, but we need to remove 
	# human-readable crap from it 
	my $sendsize = pop(@rawsize);
	$sendsize =~ s/^size\s*//;
	chomp $sendsize;

	# to avoid confusion with a zero size pv, give sendsize
	# a minimum 4K value - or if empty, make sure it reads UNKNOWN
	if ($debug) { print "DEBUG: sendsize = $sendsize\n"; }
	if ($sendsize eq '' || $exit != 0) {
		$sendsize = '0';
	} elsif ($sendsize < 4096) { 
		$sendsize = 4096; 
	}
	return $sendsize;
}

sub getdate {
	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	$year += 1900;
	my %date;
	$date{'unix'} = (((((((($year - 1971) * 365) + $yday) * 24) + $hour) * 60) + $min) * 60) + $sec;
	$date{'year'} = $year;
	$date{'sec'} = sprintf ("%02u", $sec);
	$date{'min'} = sprintf ("%02u", $min);
	$date{'hour'} = sprintf ("%02u", $hour);
	$date{'mday'} = sprintf ("%02u", $mday);
	$date{'mon'} = sprintf ("%02u", ($mon + 1));
	$date{'stamp'} = "$date{'year'}-$date{'mon'}-$date{'mday'}:$date{'hour'}:$date{'min'}:$date{'sec'}";
	return %date;
}