Add GetOptions helptext to syncoid

Lots of changes here
Notably:
nocommandcheck is now no-command-check
sshoptions can be specified multiple times
sshcipher now defaults to whatever your ssh client wants to
compress no longer accepts "no" or "0"
This commit is contained in:
Charles Pigott 2017-08-15 15:56:04 +01:00
parent 36980d4788
commit f8ea8f907d
1 changed files with 139 additions and 180 deletions

319
syncoid
View File

@ -4,25 +4,43 @@
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
my $version = '1.4.16';
$::VERSION = '1.4.16';
use strict;
use warnings;
use Data::Dumper;
use Getopt::Long qw(:config auto_version auto_help);
use Pod::Usage;
use Time::Local;
use Sys::Hostname;
my %args = getargs(@ARGV);
# Blank defaults to use ssh client's default
# TODO: Merge into a single "sshflags" option?
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
"debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2);
if ($args{'version'}) {
print "Syncoid version: $version\n";
exit 0;
$args{'compress'} = compressargset($args{'compress'} || 'default'); # Can't be done with GetOptions arg, as default still needs to be set
# TODO Expand to accept multiple sources?
if (scalar(@ARGV) != 2) {
print("Source or target not found!\n");
pod2usage(2);
exit 127;
} else {
$args{'source'} = $ARGV[0];
$args{'target'} = $ARGV[1];
}
if (!(defined $args{'source'} && defined $args{'target'})) {
print 'usage: syncoid [src_user@src_host:]src_pool/src_dataset [dst_user@dst_host:]dst_pool/dst_dataset'."\n";
exit 127;
# Could possibly merge these into an options function
if (length $args{'source-bwlimit'}) {
$args{'source-bwlimit'} = "-R $args{'source-bwlimit'}";
}
if (length $args{'target-bwlimit'}) {
$args{'target-bwlimit'} = "-r $args{'target-bwlimit'}";
}
$args{'streamarg'} = (defined $args{'no-stream'} ? '-i' : '-I');
my $rawsourcefs = $args{'source'};
my $rawtargetfs = $args{'target'};
@ -32,25 +50,8 @@ my $quiet = $args{'quiet'};
my $zfscmd = '/sbin/zfs';
my $sshcmd = '/usr/bin/ssh';
my $pscmd = '/bin/ps';
my $sshcipher;
if (defined $args{'c'}) {
$sshcipher = "-c $args{'c'}";
} else {
$sshcipher = '-c chacha20-poly1305@openssh.com,arcfour';
}
my $sshport = '-p 22';
my $sshoption;
if (defined $args{'o'}) {
my @options = split(',', $args{'o'});
foreach my $option (@options) {
$sshoption .= " -o $option";
if ($option eq "NoneSwitch=yes") {
$sshcipher = "";
}
}
} else {
$sshoption = "";
}
my $pvcmd = '/usr/bin/pv';
my $mbuffercmd = '/usr/bin/mbuffer';
my $sudocmd = '/usr/bin/sudo';
@ -59,23 +60,24 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null';
# being present on remote machines.
my $lscmd = '/bin/ls';
if ( $args{'sshport'} ) {
$sshport = "-p $args{'sshport'}";
if (length $args{'sshcipher'}) {
$args{'sshcipher'} = "-c $args{'sshcipher'}";
}
if (length $args{'sshport'}) {
$args{'sshport'} = "-p $args{'sshport'}";
}
if (length $args{'sshkey'}) {
$args{'sshkey'} = "-i $args{'sshkey'}";
}
my $sshoptions = join " ", map { "-o " . $_ } $args{'sshoption'};
# figure out if source and/or target are remote.
if ( $args{'sshkey'} ) {
$sshcmd = "$sshcmd $sshoption $sshcipher $sshport -i $args{'sshkey'}";
}
else {
$sshcmd = "$sshcmd $sshoption $sshcipher $sshport";
}
$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}";
my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs);
my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs);
my $sourcesudocmd;
my $targetsudocmd;
if ($sourceisroot) { $sourcesudocmd = ''; } else { $sourcesudocmd = $sudocmd; }
if ($targetisroot) { $targetsudocmd = ''; } else { $targetsudocmd = $sudocmd; }
my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
my $targetsudocmd = $targetisroot ? '' : $sudocmd;
# figure out whether compression, mbuffering, pv
# are available on source, target, local machines.
@ -88,7 +90,7 @@ my %snaps;
## can loop across children separately, for recursive ##
## replication ##
if (! $args{'recursive'}) {
if (defined $args{'recursive'}) {
syncdataset($sourcehost, $sourcefs, $targethost, $targetfs);
} else {
if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; }
@ -161,11 +163,15 @@ sub syncdataset {
%snaps = (%sourcesnaps, %targetsnaps);
}
if ($args{'dumpsnaps'}) { print "merged snapshot list of $targetfs: \n"; dumphash(\%snaps); print "\n\n\n"; }
if (defined $args{'dumpsnaps'}) {
print "merged snapshot list of $targetfs: \n";
dumphash(\%snaps);
print "\n\n\n";
}
# create a new syncoid snapshot on the source filesystem.
my $newsyncsnap;
if (!defined ($args{'no-sync-snap'}) ) {
if (!defined $args{'no-sync-snap'}) {
$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
} else {
# we don't want sync snapshots created, so use the newest snapshot we can find.
@ -334,111 +340,38 @@ sub syncdataset {
} # end syncdataset()
sub getargs {
my @args = @_;
my %args;
my %novaluearg;
my %validarg;
push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','c','o','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap');
foreach my $item (@validargs) { $validarg{$item} = 1; }
push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap');
foreach my $item (@novalueargs) { $novaluearg{$item} = 1; }
while (my $rawarg = shift(@args)) {
my $arg = $rawarg;
my $argvalue = '';
if ($rawarg =~ /=/) {
# user specified the value for a CLI argument with =
# instead of with blank space. separate appropriately.
$argvalue = $arg;
$arg =~ s/=.*$//;
$argvalue =~ s/^.*=//;
}
if ($rawarg =~ /^--/) {
# doubledash arg
$arg =~ s/^--//;
if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
if ($novaluearg{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} elsif ($arg =~ /^-/) {
# singledash arg
$arg =~ s/^-//;
if (! $validarg{$arg}) { die "ERROR: don't understand argument $rawarg.\n"; }
if ($novaluearg{$arg}) {
$args{$arg} = 1;
} else {
# if this CLI arg takes a user-specified value and
# we don't already have it, then the user must have
# specified with a space, so pull in the next value
# from the array as this value rather than as the
# next argument.
if ($argvalue eq '') { $argvalue = shift(@args); }
$args{$arg} = $argvalue;
}
} else {
# bare arg
if (defined $args{'source'}) {
if (! defined $args{'target'}) {
$args{'target'} = $arg;
} else {
die "ERROR: don't know what to do with third bare argument $rawarg.\n";
}
} else {
$args{'source'} = $arg;
}
}
sub compressargset {
my ($value) = @_;
my %comargs = ('rawcmd' => '', 'args' => '', 'decomrawcmd' => '', 'decomargs' => '');
if (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) {
warn "Unrecognised compression value $value, defaulting to lzo";
$value = 'default';
}
if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; }
if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; }
if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; }
if ($args{'r'}) { $args{'recursive'} = $args{'r'}; }
if (!defined $args{'compress'}) { $args{'compress'} = 'default'; }
if ($args{'compress'} eq 'gzip') {
$args{'rawcompresscmd'} = '/bin/gzip';
$args{'compressargs'} = '-3';
$args{'rawdecompresscmd'} = '/bin/zcat';
$args{'decompressargs'} = '';
} elsif ( ($args{'compress'} eq 'pigz-fast')) {
$args{'rawcompresscmd'} = '/usr/bin/pigz';
$args{'compressargs'} = '-3';
$args{'rawdecompresscmd'} = '/usr/bin/pigz';
$args{'decompressargs'} = '-dc';
} elsif ( ($args{'compress'} eq 'pigz-slow')) {
$args{'rawcompresscmd'} = '/usr/bin/pigz';
$args{'compressargs'} = '-9';
$args{'rawdecompresscmd'} = '/usr/bin/pigz';
$args{'decompressargs'} = '-dc';
} elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) {
$args{'rawcompresscmd'} = '/usr/bin/lzop';
$args{'compressargs'} = '';
$args{'rawdecompresscmd'} = '/usr/bin/lzop';
$args{'decompressargs'} = '-dfc';
} else {
$args{'rawcompresscmd'} = '';
$args{'compressargs'} = '';
$args{'rawdecompresscmd'} = '';
$args{'decompressargs'} = '';
if ($value eq 'gzip') {
$comargs{'rawcmd'} = '/bin/gzip';
$comargs{'args'} = '-3';
$comargs{'decomrawcmd'} = '/bin/zcat';
$comargs{'decomargs'} = '';
} elsif ($value eq 'pigz-fast') {
$comargs{'rawcmd'} = '/usr/bin/pigz';
$comargs{'args'} = '-3';
$comargs{'decomrawcmd'} = '/usr/bin/pigz';
$comargs{'decomargs'} = '-dc';
} elsif ($value eq 'pigz-slow') {
$comargs{'rawcmd'} = '/usr/bin/pigz';
$comargs{'args'} = '-9';
$comargs{'decomrawcmd'} = '/usr/bin/pigz';
$comargs{'decomargs'} = '-dc';
} elsif (($value eq 'lzo') || ($value eq 'default') ) {
$comargs{'rawcmd'} = '/usr/bin/lzop';
$comargs{'args'} = '';
$comargs{'decomrawcmd'} = '/usr/bin/lzop';
$comargs{'decomargs'} = '-dfc';
}
$args{'compresscmd'} = "$args{'rawcompresscmd'} $args{'compressargs'}";
$args{'decompresscmd'} = "$args{'rawdecompresscmd'} $args{'decompressargs'}";
return %args;
$comargs{'compress'} = $value;
$comargs{'cmd'} = "$comargs{'rawcmd'} $comargs{'args'}";
$comargs{'decomcmd'} = "$comargs{'decomrawcmd'} $comargs{'decomargs'}";
return \%comargs;
}
sub checkcommands {
@ -468,24 +401,15 @@ sub checkcommands {
# if raw compress command is null, we must have specified no compression. otherwise,
# make sure that compression is available everywhere we need it
if ($args{'rawcompresscmd'} eq '') {
$avail{'sourcecompress'} = 0;
$avail{'sourcecompress'} = 0;
$avail{'localcompress'} = 0;
if ($args{'compress'} eq 'none' ||
$args{'compress'} eq 'no' ||
$args{'compress'} eq '0') {
if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; }
} else {
print "WARN: value $args{'compress'} for argument --compress not understood, proceeding without compression.\n";
}
if ($args{'compress'}{'compress'} eq 'none') {
if ($debug) { print "DEBUG: compression forced off from command line arguments.\n"; }
} else {
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on source...\n"; }
$avail{'sourcecompress'} = `$sourcessh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on target...\n"; }
$avail{'targetcompress'} = `$targetssh $lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'rawcompresscmd'} on local machine...\n"; }
$avail{'localcompress'} = `$lscmd $args{'rawcompresscmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on source...\n"; }
$avail{'sourcecompress'} = `$sourcessh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on target...\n"; }
$avail{'targetcompress'} = `$targetssh $lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
if ($debug) { print "DEBUG: checking availability of $args{'compress'}{'rawcmd'} on local machine...\n"; }
$avail{'localcompress'} = `$lscmd $args{'compress'}{'rawcmd'} 2>/dev/null`;
}
my ($s,$t);
@ -511,14 +435,14 @@ sub checkcommands {
if ($avail{'sourcecompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on source $s- sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
if ($avail{'targetcompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on target $t - sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on target $t - sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
@ -530,8 +454,8 @@ sub checkcommands {
# corner case - if source AND target are BOTH remote, we have to check for local compress too
if ($sourcehost ne '' && $targethost ne '' && $avail{'localcompress'} eq '') {
if ($args{'rawcompresscmd'} ne '') {
print "WARN: $args{'compresscmd'} not available on local machine - sync will continue without compression.\n";
if ($args{'compress'}{'rawcmd'} ne '') {
print "WARN: $args{'compress'}{'rawcmd'} not available on local machine - sync will continue without compression.\n";
}
$avail{'compress'} = 0;
}
@ -687,9 +611,9 @@ sub buildsynccmd {
$synccmd = "$sendcmd |";
# avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here
my $bwlimit = '';
if (defined $args{'source-bwlimit'}) {
if (length $args{'bwlimit'}) {
$bwlimit = $args{'source-bwlimit'};
} elsif (defined $args{'target-bwlimit'}) {
} elsif (length $args{'target-bwlimit'}) {
$bwlimit = $args{'target-bwlimit'};
}
@ -698,18 +622,18 @@ sub buildsynccmd {
$synccmd .= " $recvcmd";
} elsif ($sourcehost eq '') {
# local source, remote target.
#$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
#$synccmd = "$sendcmd | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'";
$synccmd = "$sendcmd |";
if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'cmd'} |"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
$synccmd .= " $sshcmd $targethost '";
if ($avail{'targetmbuffer'}) { $synccmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'decompresscmd'} |"; }
if ($avail{'compress'}) { $synccmd .= " $args{'compress'}{'decomcmd'} |"; }
$synccmd .= " $recvcmd'";
} elsif ($targethost eq '') {
# remote source, local target.
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $mbuffercmd | $pvcmd | $recvcmd";
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'decompress'}{'cmd'} | $mbuffercmd | $pvcmd | $recvcmd";
$synccmd = "$sshcmd $sourcehost '$sendcmd";
if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
@ -720,18 +644,18 @@ sub buildsynccmd {
$synccmd .= "$recvcmd";
} else {
#remote source, remote target... weird, but whatever, I'm not here to judge you.
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compresscmd'} | $mbuffercmd' | $args{'decompresscmd'} | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'";
#$synccmd = "$sshcmd $sourcehost '$sendcmd | $args{'compress'}{'cmd'} | $mbuffercmd' | $args{'compress'}{'decomcmd'} | $pvcmd | $args{'compress'}{'cmd'} | $mbuffercmd | $sshcmd $targethost '$args{'compress'}{'decomcmd'} | $mbuffercmd | $recvcmd'";
$synccmd = "$sshcmd $sourcehost '$sendcmd";
if ($avail{'compress'}) { $synccmd .= " | $args{'compresscmd'}"; }
if ($avail{'compress'}) { $synccmd .= " | $args{'compress'}{'cmd'}"; }
if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
$synccmd .= "' | ";
if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; }
if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'cmd'} | "; }
if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; }
$synccmd .= "$sshcmd $targethost '";
if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; }
if ($avail{'compress'}) { $synccmd .= "$args{'compress'}{'decomcmd'} | "; }
$synccmd .= "$recvcmd'";
}
return $synccmd;
@ -865,7 +789,7 @@ sub getssh {
if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; }
# now we need to establish a persistent master SSH connection
$socket = "/tmp/syncoid-$remoteuser-$rhost-" . time();
open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |";
open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |";
close FH;
$rhost = "-S $socket $rhost";
} else {
@ -985,4 +909,39 @@ sub getdate {
return %date;
}
__END__
=head1 NAME
syncoid - ZFS snapshot replication tool
=head1 SYNOPSIS
syncoid [options]... SOURCE TARGET
or syncoid [options]... SOURCE [USER@]HOST:TARGET
or syncoid [options]... [USER@]HOST:SOURCE [TARGET]
or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET
SOURCE Source ZFS dataset. Can be either local or remote
TARGET Target ZFS dataset. Can be either local or remote
Options:
--compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none
--recursive|r Also transfers child datasets
--source-bwlimit=<limit k|m|g|t> Bandwidth limit on the source transfer
--target-bwlimit=<limit k|m|g|t> Bandwidth limit on the target transfer
--sshkey=FILE Specifies a ssh public key to use to connect
--sshport=PORT Connects to remote on a particular port
--sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set
--sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times
--help Prints this helptext
--verbose Prints the version number
--debug Prints out a lot of additional information during a syncoid run
--monitor-version Currently does nothing
--quiet Suppresses non-error output
--dumpsnaps Dumps a list of snapshots during the run
--no-command-checks Do not check command existence before attempting transfer. Not recommended