Merge remote-tracking branch 'upstream/master' into extra-identifier
This commit is contained in:
commit
b32d5a6833
2
INSTALL
2
INSTALL
|
@ -8,7 +8,7 @@ default for SSH transport since v1.4.6. Syncoid runs will fail if one of them
|
|||
is not available on either end of the transport.
|
||||
|
||||
On Ubuntu: apt install pv lzop mbuffer
|
||||
On CentOS: yum install lzo pv mbuffer lzop
|
||||
On CentOS: yum install lzo pv mbuffer lzop perl-Data-Dumper
|
||||
On FreeBSD: pkg install pv mbuffer lzop
|
||||
|
||||
FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than
|
||||
|
|
16
README.md
16
README.md
|
@ -6,9 +6,11 @@
|
|||
|
||||
More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job:
|
||||
```
|
||||
* * * * * /usr/local/bin/sanoid --cron
|
||||
* * * * * TZ=UTC /usr/local/bin/sanoid --cron
|
||||
```
|
||||
|
||||
`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times
|
||||
|
||||
And its /etc/sanoid/sanoid.conf might look something like this:
|
||||
|
||||
```
|
||||
|
@ -62,6 +64,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da
|
|||
|
||||
This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file.
|
||||
|
||||
+ --monitor-capacity
|
||||
|
||||
This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file.
|
||||
|
||||
+ --force-update
|
||||
|
||||
This clears out sanoid's zfs snapshot listing cache. This is normally not needed.
|
||||
|
@ -130,6 +136,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
|
|||
|
||||
This will also transfer child datasets.
|
||||
|
||||
+ --skip-parent
|
||||
|
||||
This will skip the syncing of the parent dataset. Does nothing without '--recursive' option.
|
||||
|
||||
+ --compress <compression type>
|
||||
|
||||
Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
|
||||
|
@ -154,6 +164,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
|
|||
|
||||
This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots.
|
||||
|
||||
+ --exclude=REGEX
|
||||
|
||||
The given regular expression will be matched against all datasets which would be synced by this run and excludes them. This argument can be specified multiple times.
|
||||
|
||||
+ --no-resume
|
||||
|
||||
This argument tells syncoid to not use resumeable zfs send/receive streams.
|
||||
|
|
|
@ -1,3 +1,18 @@
|
|||
sanoid (1.4.18) unstable; urgency=medium
|
||||
|
||||
implemented special character handling and support of ZFS resume/receive tokens by default in syncoid,
|
||||
thank you @phreaker0!
|
||||
|
||||
-- Jim Salter <github@jrs-s.net> Wed, 25 Apr 2018 16:24:00 -0400
|
||||
|
||||
sanoid (1.4.17) unstable; urgency=medium
|
||||
|
||||
changed die to warn when unexpectedly unable to remove a snapshot - this
|
||||
allows sanoid to continue taking/removing other snapshots not affected by
|
||||
whatever lock prevented the first from being taken or removed
|
||||
|
||||
-- Jim Salter <github@jrs-s.net> Wed, 8 Nov 2017 15:25:00 -0400
|
||||
|
||||
sanoid (1.4.16) unstable; urgency=medium
|
||||
|
||||
* merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's
|
||||
|
|
|
@ -5,5 +5,6 @@ After=zfs.target
|
|||
ConditionFileNotEmpty=/etc/sanoid/sanoid.conf
|
||||
|
||||
[Service]
|
||||
Environment=TZ=UTC
|
||||
Type=oneshot
|
||||
ExecStart=/usr/sbin/sanoid --cron
|
||||
|
|
|
@ -58,6 +58,7 @@ Requires=zfs.target
|
|||
After=zfs.target
|
||||
|
||||
[Service]
|
||||
Environment=TZ=UTC
|
||||
Type=oneshot
|
||||
ExecStart=%{_sbindir}/sanoid --cron
|
||||
EOF
|
||||
|
|
267
sanoid
267
sanoid
|
@ -18,7 +18,8 @@ use Time::Local; # to parse dates in reverse
|
|||
my %args = ("configdir" => "/etc/sanoid");
|
||||
GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet",
|
||||
"monitor-health", "force-update", "configdir=s",
|
||||
"monitor-snapshots", "take-snapshots", "prune-snapshots"
|
||||
"monitor-snapshots", "take-snapshots", "prune-snapshots",
|
||||
"monitor-capacity"
|
||||
) or pod2usage(2);
|
||||
|
||||
# If only config directory (or nothing) has been specified, default to --cron --verbose
|
||||
|
@ -39,8 +40,10 @@ my %config = init($conf_file,$default_conf_file);
|
|||
|
||||
# if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL
|
||||
my $forcecacheupdate = 0;
|
||||
my $cache = '/var/cache/sanoidsnapshots.txt';
|
||||
my $cacheTTL = 900; # 15 minutes
|
||||
my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate );
|
||||
my %pruned;
|
||||
|
||||
my %snapsbytype = getsnapsbytype( \%config, \%snaps );
|
||||
|
||||
|
@ -52,6 +55,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath );
|
|||
if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); }
|
||||
if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); }
|
||||
if ($args{'monitor-health'}) { monitor_health(@params); }
|
||||
if ($args{'monitor-capacity'}) { monitor_capacity(@params); }
|
||||
if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); }
|
||||
|
||||
if ($args{'cron'}) {
|
||||
|
@ -174,6 +178,61 @@ sub monitor_snapshots {
|
|||
exit $errorlevel;
|
||||
}
|
||||
|
||||
|
||||
####################################################################################
|
||||
####################################################################################
|
||||
####################################################################################
|
||||
|
||||
sub monitor_capacity {
|
||||
my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
|
||||
my %pools;
|
||||
my @messages;
|
||||
my $errlevel=0;
|
||||
|
||||
# build pool list with corresponding capacity limits
|
||||
foreach my $section (keys %config) {
|
||||
my @pool = split ('/',$section);
|
||||
|
||||
if (scalar @pool == 1 || !defined($pools{$pool[0]}) ) {
|
||||
my %capacitylimits;
|
||||
|
||||
if (!check_capacity_limit($config{$section}{'capacity_warn'})) {
|
||||
die "ERROR: invalid zpool capacity warning limit!\n";
|
||||
}
|
||||
|
||||
if ($config{$section}{'capacity_warn'} != 0) {
|
||||
$capacitylimits{'warn'} = $config{$section}{'capacity_warn'};
|
||||
}
|
||||
|
||||
if (!check_capacity_limit($config{$section}{'capacity_crit'})) {
|
||||
die "ERROR: invalid zpool capacity critical limit!\n";
|
||||
}
|
||||
|
||||
if ($config{$section}{'capacity_crit'} != 0) {
|
||||
$capacitylimits{'crit'} = $config{$section}{'capacity_crit'};
|
||||
}
|
||||
|
||||
if (%capacitylimits) {
|
||||
$pools{$pool[0]} = \%capacitylimits;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach my $pool (keys %pools) {
|
||||
my $capacitylimitsref = $pools{$pool};
|
||||
|
||||
my ($exitcode, $msg) = check_zpool_capacity($pool,\%$capacitylimitsref);
|
||||
if ($exitcode > $errlevel) { $errlevel = $exitcode; }
|
||||
chomp $msg;
|
||||
push (@messages, $msg);
|
||||
}
|
||||
|
||||
my @warninglevels = ('','*** WARNING *** ','*** CRITICAL *** ');
|
||||
my $message = $warninglevels[$errlevel] . join (', ',@messages);
|
||||
print "$message\n";
|
||||
exit $errlevel;
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
####################################################################################
|
||||
####################################################################################
|
||||
|
@ -235,23 +294,30 @@ sub prune_snapshots {
|
|||
foreach my $snap( @prunesnaps ){
|
||||
if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
|
||||
if (iszfsbusy($path)) {
|
||||
print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n";
|
||||
if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; }
|
||||
} else {
|
||||
if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; }
|
||||
if (! $args{'readonly'}) {
|
||||
if (system($zfs, "destroy", $snap) == 0) {
|
||||
$pruned{$snap} = 1;
|
||||
} else {
|
||||
warn "could not remove $snap : $?";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
removelock('sanoid_pruning');
|
||||
$forcecacheupdate = 1;
|
||||
%snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate);
|
||||
removecachedsnapshots(0);
|
||||
} else {
|
||||
print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n";
|
||||
if ($args{'verbose'}) { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# if there were any deferred cache updates,
|
||||
# do them now and wait if necessary
|
||||
removecachedsnapshots(1);
|
||||
} # end prune_snapshots
|
||||
|
||||
|
||||
|
@ -268,6 +334,19 @@ sub take_snapshots {
|
|||
|
||||
my @newsnaps;
|
||||
|
||||
# get utc timestamp of the current day for DST check
|
||||
my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'});
|
||||
my ($isdst) = (localtime($daystartUtc))[8];
|
||||
my $dstOffset = 0;
|
||||
|
||||
if ($isdst ne $datestamp{'isdst'}) {
|
||||
# current dst is different then at the beginning og the day
|
||||
if ($isdst) {
|
||||
# DST ended in the current day
|
||||
$dstOffset = 60*60;
|
||||
}
|
||||
}
|
||||
|
||||
if ($args{'verbose'}) { print "INFO: taking snapshots...\n"; }
|
||||
foreach my $section (keys %config) {
|
||||
if ($section =~ /^template/) { next; }
|
||||
|
@ -291,6 +370,9 @@ sub take_snapshots {
|
|||
my @preferredtime;
|
||||
my $lastpreferred;
|
||||
|
||||
# to avoid duplicates with DST
|
||||
my $dateSuffix = "";
|
||||
|
||||
if ($type eq 'hourly') {
|
||||
push @preferredtime,0; # try to hit 0 seconds
|
||||
push @preferredtime,$config{$section}{'hourly_min'};
|
||||
|
@ -299,6 +381,13 @@ sub take_snapshots {
|
|||
push @preferredtime,($datestamp{'mon'}-1); # january is month 0
|
||||
push @preferredtime,$datestamp{'year'};
|
||||
$lastpreferred = timelocal(@preferredtime);
|
||||
|
||||
if ($dstOffset ne 0) {
|
||||
# timelocal doesn't take DST into account
|
||||
$lastpreferred += $dstOffset;
|
||||
# DST ended, avoid duplicates
|
||||
$dateSuffix = "_y";
|
||||
}
|
||||
if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's
|
||||
} elsif ($type eq 'daily') {
|
||||
push @preferredtime,0; # try to hit 0 seconds
|
||||
|
@ -308,7 +397,29 @@ sub take_snapshots {
|
|||
push @preferredtime,($datestamp{'mon'}-1); # january is month 0
|
||||
push @preferredtime,$datestamp{'year'};
|
||||
$lastpreferred = timelocal(@preferredtime);
|
||||
if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's
|
||||
|
||||
# timelocal doesn't take DST into account
|
||||
$lastpreferred += $dstOffset;
|
||||
|
||||
# check if the planned time has different DST flag than the current
|
||||
my ($isdst) = (localtime($lastpreferred))[8];
|
||||
if ($isdst ne $datestamp{'isdst'}) {
|
||||
if (!$isdst) {
|
||||
# correct DST difference
|
||||
$lastpreferred -= 60*60;
|
||||
}
|
||||
}
|
||||
|
||||
if ($lastpreferred > time()) {
|
||||
$lastpreferred -= 60*60*24;
|
||||
|
||||
if ($dstOffset ne 0) {
|
||||
# because we are going back one day
|
||||
# the DST difference has to be accounted
|
||||
# for in reverse now
|
||||
$lastpreferred -= 2*$dstOffset;
|
||||
}
|
||||
} # preferred time is later today - so look at yesterday's
|
||||
} elsif ($type eq 'monthly') {
|
||||
push @preferredtime,0; # try to hit 0 seconds
|
||||
push @preferredtime,$config{$section}{'monthly_min'};
|
||||
|
@ -336,7 +447,7 @@ sub take_snapshots {
|
|||
# update to most current possible datestamp
|
||||
%datestamp = get_date();
|
||||
# print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n";
|
||||
push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type");
|
||||
push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -484,7 +595,6 @@ sub getsnaps {
|
|||
|
||||
my ($config, $cacheTTL, $forcecacheupdate) = @_;
|
||||
|
||||
my $cache = '/var/cache/sanoidsnapshots.txt';
|
||||
my @rawsnaps;
|
||||
|
||||
my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache);
|
||||
|
@ -521,7 +631,7 @@ sub getsnaps {
|
|||
}
|
||||
|
||||
foreach my $snap (@rawsnaps) {
|
||||
my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/);
|
||||
my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\t*creation\t*(\d*)/);
|
||||
|
||||
# avoid pissing off use warnings
|
||||
if (defined $snapname) {
|
||||
|
@ -866,6 +976,11 @@ sub check_zpool() {
|
|||
## other cases
|
||||
my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/;
|
||||
|
||||
if (!defined($sta)) {
|
||||
# cache and logs are special and don't have a status
|
||||
next;
|
||||
}
|
||||
|
||||
## pool online, not degraded thanks to dead/corrupted disk
|
||||
if ($state eq "OK" && $sta eq "UNAVAIL") {
|
||||
$state="WARNING";
|
||||
|
@ -900,6 +1015,74 @@ sub check_zpool() {
|
|||
return ($ERRORS{$state},$msg);
|
||||
} # end check_zpool()
|
||||
|
||||
sub check_capacity_limit() {
|
||||
my $value = shift;
|
||||
|
||||
if (!defined($value) || $value !~ /^\d+\z/) {
|
||||
return undef;
|
||||
}
|
||||
|
||||
if ($value < 0 || $value > 100) {
|
||||
return undef;
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
sub check_zpool_capacity() {
|
||||
my %ERRORS=('DEPENDENT'=>4,'UNKNOWN'=>3,'OK'=>0,'WARNING'=>1,'CRITICAL'=>2);
|
||||
my $state="UNKNOWN";
|
||||
my $msg="FAILURE";
|
||||
|
||||
my $pool=shift;
|
||||
my $capacitylimitsref=shift;
|
||||
my %capacitylimits=%$capacitylimitsref;
|
||||
|
||||
my $statcommand="/sbin/zpool list -H -o cap $pool";
|
||||
|
||||
if (! open STAT, "$statcommand|") {
|
||||
print ("$state '$statcommand' command returns no result!\n");
|
||||
exit $ERRORS{$state};
|
||||
}
|
||||
|
||||
my $line = <STAT>;
|
||||
close(STAT);
|
||||
|
||||
chomp $line;
|
||||
my @row = split(/ +/, $line);
|
||||
my $cap=$row[0];
|
||||
|
||||
## check for valid capacity value
|
||||
if ($cap !~ m/^[0-9]{1,3}%$/ ) {
|
||||
$state = "CRITICAL";
|
||||
$msg = sprintf "ZPOOL {%s} does not exist and/or is not responding!\n", $pool;
|
||||
print $state, " ", $msg;
|
||||
exit ($ERRORS{$state});
|
||||
}
|
||||
|
||||
$state="OK";
|
||||
|
||||
# check capacity
|
||||
my $capn = $cap;
|
||||
$capn =~ s/\D//g;
|
||||
|
||||
if (defined($capacitylimits{"warn"})) {
|
||||
if ($capn >= $capacitylimits{"warn"}) {
|
||||
$state = "WARNING";
|
||||
}
|
||||
}
|
||||
|
||||
if (defined($capacitylimits{"crit"})) {
|
||||
if ($capn >= $capacitylimits{"crit"}) {
|
||||
$state = "CRITICAL";
|
||||
}
|
||||
}
|
||||
|
||||
$msg = sprintf "ZPOOL %s : %s\n", $pool, $cap;
|
||||
$msg = "$state $msg";
|
||||
return ($ERRORS{$state},$msg);
|
||||
} # end check_zpool_capacity()
|
||||
|
||||
######################################################################################################
|
||||
######################################################################################################
|
||||
######################################################################################################
|
||||
|
@ -930,13 +1113,22 @@ sub checklock {
|
|||
# no lockfile
|
||||
return 1;
|
||||
}
|
||||
# make sure lockfile contains something
|
||||
if ( -z $lockfile) {
|
||||
# zero size lockfile, something is wrong
|
||||
die "ERROR: something is wrong! $lockfile is empty\n";
|
||||
}
|
||||
|
||||
# lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if
|
||||
# there's still a process running with that pid and with the same mutex.
|
||||
|
||||
open FH, "< $lockfile";
|
||||
open FH, "< $lockfile" or die "ERROR: unable to open $lockfile";
|
||||
my @lock = <FH>;
|
||||
close FH;
|
||||
# if we didn't get exactly 2 items from the lock file there is a problem
|
||||
if (scalar(@lock) != 2) {
|
||||
die "ERROR: $lockfile is invalid.\n"
|
||||
}
|
||||
|
||||
my $lockmutex = pop(@lock);
|
||||
my $lockpid = pop(@lock);
|
||||
|
@ -948,7 +1140,6 @@ sub checklock {
|
|||
# we own the lockfile. no need to check any further.
|
||||
return 2;
|
||||
}
|
||||
|
||||
open PL, "$pscmd -p $lockpid -o args= |";
|
||||
my @processlist = <PL>;
|
||||
close PL;
|
||||
|
@ -1056,6 +1247,55 @@ sub getchilddatasets {
|
|||
return @children;
|
||||
}
|
||||
|
||||
#######################################################################################################################3
|
||||
#######################################################################################################################3
|
||||
#######################################################################################################################3
|
||||
|
||||
sub removecachedsnapshots {
|
||||
my $wait = shift;
|
||||
|
||||
if (not %pruned) {
|
||||
return;
|
||||
}
|
||||
|
||||
my $unlocked = checklock('sanoid_cacheupdate');
|
||||
|
||||
if ($wait != 1 && not $unlocked) {
|
||||
if ($args{'verbose'}) { print "INFO: deferring cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; }
|
||||
return;
|
||||
}
|
||||
|
||||
# wait until we can get a lock to do our cache changes
|
||||
while (not $unlocked) {
|
||||
if ($args{'verbose'}) { print "INFO: waiting for cache update lock held by another sanoid process.\n"; }
|
||||
sleep(10);
|
||||
$unlocked = checklock('sanoid_cacheupdate');
|
||||
}
|
||||
|
||||
writelock('sanoid_cacheupdate');
|
||||
|
||||
if ($args{'verbose'}) {
|
||||
print "INFO: removing destroyed snapshots from cache.\n";
|
||||
}
|
||||
open FH, "< $cache";
|
||||
my @rawsnaps = <FH>;
|
||||
close FH;
|
||||
|
||||
open FH, "> $cache" or die 'Could not write to $cache!\n';
|
||||
foreach my $snapline ( @rawsnaps ) {
|
||||
my @columns = split("\t", $snapline);
|
||||
my $snap = $columns[0];
|
||||
print FH $snapline unless ( exists($pruned{$snap}) );
|
||||
}
|
||||
close FH;
|
||||
|
||||
removelock('sanoid_cacheupdate');
|
||||
%snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate);
|
||||
|
||||
# clear hash
|
||||
undef %pruned;
|
||||
}
|
||||
|
||||
__END__
|
||||
|
||||
=head1 NAME
|
||||
|
@ -1079,6 +1319,7 @@ Options:
|
|||
--force-update Clears out sanoid's zfs snapshot cache
|
||||
|
||||
--monitor-health Reports on zpool "health", in a Nagios compatible format
|
||||
--monitor-capacity Reports on zpool capacity, in a Nagios compatible format
|
||||
--monitor-snapshots Reports on snapshot "health", in a Nagios compatible format
|
||||
--take-snapshots Creates snapshots as specified in sanoid.conf
|
||||
--prune-snapshots Purges expired snapshots as specified in sanoid.conf
|
||||
|
|
|
@ -70,3 +70,7 @@ monthly_warn = 32
|
|||
monthly_crit = 35
|
||||
yearly_warn = 0
|
||||
yearly_crit = 0
|
||||
|
||||
# default limits for capacity checks (if set to 0, limit will not be checked)
|
||||
capacity_warn = 80
|
||||
capacity_crit = 95
|
||||
|
|
49
syncoid
49
syncoid
|
@ -19,7 +19,7 @@ use Sys::Hostname;
|
|||
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
|
||||
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
|
||||
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
|
||||
"debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "identifier=s") or pod2usage(2);
|
||||
"debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2);
|
||||
|
||||
my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
|
||||
|
||||
|
@ -152,6 +152,25 @@ sub getchilddatasets {
|
|||
my @children = <FH>;
|
||||
close FH;
|
||||
|
||||
if (defined $args{'skip-parent'}) {
|
||||
# parent dataset is the first element
|
||||
shift @children;
|
||||
}
|
||||
|
||||
if (defined $args{'exclude'}) {
|
||||
my $excludes = $args{'exclude'};
|
||||
foreach (@$excludes) {
|
||||
for my $i ( 0 .. $#children ) {
|
||||
if ($children[$i] =~ /$_/) {
|
||||
if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; }
|
||||
undef $children[$i]
|
||||
}
|
||||
}
|
||||
|
||||
@children = grep{ defined }@children;
|
||||
}
|
||||
}
|
||||
|
||||
return @children;
|
||||
}
|
||||
|
||||
|
@ -434,6 +453,18 @@ sub compressargset {
|
|||
decomrawcmd => '/usr/bin/pigz',
|
||||
decomargs => '-dc',
|
||||
},
|
||||
'zstd-fast' => {
|
||||
rawcmd => '/usr/bin/zstd',
|
||||
args => '-3',
|
||||
decomrawcmd => '/usr/bin/zstd',
|
||||
decomargs => '-dc',
|
||||
},
|
||||
'zstd-slow' => {
|
||||
rawcmd => '/usr/bin/zstd',
|
||||
args => '-19',
|
||||
decomrawcmd => '/usr/bin/zstd',
|
||||
decomargs => '-dc',
|
||||
},
|
||||
'lzo' => {
|
||||
rawcmd => '/usr/bin/lzop',
|
||||
args => '',
|
||||
|
@ -444,7 +475,7 @@ sub compressargset {
|
|||
|
||||
if ($value eq 'default') {
|
||||
$value = $DEFAULT_COMPRESSION;
|
||||
} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) {
|
||||
} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) {
|
||||
warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION";
|
||||
$value = $DEFAULT_COMPRESSION;
|
||||
}
|
||||
|
@ -675,7 +706,7 @@ sub getzfsvalue {
|
|||
open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |";
|
||||
my $value = <FH>;
|
||||
close FH;
|
||||
my @values = split(/\s/,$value);
|
||||
my @values = split(/\t/,$value);
|
||||
$value = $values[2];
|
||||
return $value;
|
||||
}
|
||||
|
@ -996,7 +1027,7 @@ sub getsnaps() {
|
|||
if ($line =~ /\Q$fs\E\@.*guid/) {
|
||||
chomp $line;
|
||||
my $guid = $line;
|
||||
$guid =~ s/^.*\sguid\s*(\d*).*/$1/;
|
||||
$guid =~ s/^.*\tguid\t*(\d*).*/$1/;
|
||||
my $snap = $line;
|
||||
$snap =~ s/^.*\@(.*)\tguid.*$/$1/;
|
||||
$snaps{$type}{$snap}{'guid'}=$guid;
|
||||
|
@ -1008,7 +1039,7 @@ sub getsnaps() {
|
|||
if ($line =~ /\Q$fs\E\@.*creation/) {
|
||||
chomp $line;
|
||||
my $creation = $line;
|
||||
$creation =~ s/^.*\screation\s*(\d*).*/$1/;
|
||||
$creation =~ s/^.*\tcreation\t*(\d*).*/$1/;
|
||||
my $snap = $line;
|
||||
$snap =~ s/^.*\@(.*)\tcreation.*$/$1/;
|
||||
$snaps{$type}{$snap}{'creation'}=$creation;
|
||||
|
@ -1067,9 +1098,9 @@ sub getsendsize {
|
|||
# the output format is different in case of
|
||||
# a resumed receive
|
||||
if (defined($receivetoken)) {
|
||||
$sendsize =~ s/.*\s([0-9]+)$/$1/;
|
||||
$sendsize =~ s/.*\t([0-9]+)$/$1/;
|
||||
} else {
|
||||
$sendsize =~ s/^size\s*//;
|
||||
$sendsize =~ s/^size\t*//;
|
||||
}
|
||||
chomp $sendsize;
|
||||
|
||||
|
@ -1149,10 +1180,12 @@ Options:
|
|||
--compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none
|
||||
--identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets.
|
||||
--recursive|r Also transfers child datasets
|
||||
--skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option.
|
||||
--source-bwlimit=<limit k|m|g|t> Bandwidth limit on the source transfer
|
||||
--target-bwlimit=<limit k|m|g|t> Bandwidth limit on the target transfer
|
||||
--no-stream Replicates using newest snapshot instead of intermediates
|
||||
--no-sync-snap Does not create new snapshot, only transfers existing
|
||||
--exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times
|
||||
|
||||
--sshkey=FILE Specifies a ssh public key to use to connect
|
||||
--sshport=PORT Connects to remote on a particular port
|
||||
|
@ -1160,7 +1193,7 @@ Options:
|
|||
--sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times
|
||||
|
||||
--help Prints this helptext
|
||||
--verbose Prints the version number
|
||||
--version Prints the version number
|
||||
--debug Prints out a lot of additional information during a syncoid run
|
||||
--monitor-version Currently does nothing
|
||||
--quiet Suppresses non-error output
|
||||
|
|
|
@ -0,0 +1,55 @@
|
|||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
# this test will take hourly, daily and monthly snapshots
|
||||
# for the whole year of 2017 in the timezone Europe/Vienna
|
||||
# sanoid is run hourly and no snapshots are pruned
|
||||
|
||||
. ../common/lib.sh
|
||||
|
||||
POOL_NAME="sanoid-test-1"
|
||||
POOL_TARGET="" # root
|
||||
RESULT="/tmp/sanoid_test_result"
|
||||
RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf"
|
||||
|
||||
# UTC timestamp of start and end
|
||||
START="1483225200"
|
||||
END="1514761199"
|
||||
|
||||
# prepare
|
||||
setup
|
||||
checkEnvironment
|
||||
disableTimeSync
|
||||
|
||||
# set timezone
|
||||
ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
|
||||
|
||||
timestamp=$START
|
||||
|
||||
mkdir -p "${POOL_TARGET}"
|
||||
truncate -s 5120M "${POOL_TARGET}"/zpool.img
|
||||
|
||||
zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img
|
||||
|
||||
function cleanUp {
|
||||
zpool export "${POOL_NAME}"
|
||||
}
|
||||
|
||||
# export pool in any case
|
||||
trap cleanUp EXIT
|
||||
|
||||
while [ $timestamp -le $END ]; do
|
||||
date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose
|
||||
timestamp=$((timestamp+3600))
|
||||
done
|
||||
|
||||
saveSnapshotList "${POOL_NAME}" "${RESULT}"
|
||||
|
||||
# hourly daily monthly
|
||||
verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}"
|
||||
|
||||
# hourly count should be 8760 but one hour get's lost because of DST
|
||||
|
||||
# daily count should be 365 but one additional daily is taken
|
||||
# because the DST change leads to a day with 25 hours
|
||||
# which will trigger an additional daily snapshot
|
|
@ -0,0 +1,10 @@
|
|||
[sanoid-test-1]
|
||||
use_template = production
|
||||
|
||||
[template_production]
|
||||
hourly = 36
|
||||
daily = 30
|
||||
monthly = 3
|
||||
yearly = 0
|
||||
autosnap = yes
|
||||
autoprune = no
|
|
@ -0,0 +1,54 @@
|
|||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
# this test will check the behaviour arround a date where DST ends
|
||||
# with hourly, daily and monthly snapshots checked in a 15 minute interval
|
||||
|
||||
# Daylight saving time 2017 in Europe/Vienna began at 02:00 on Sunday, 26 March
|
||||
# and ended at 03:00 on Sunday, 29 October. All times are in
|
||||
# Central European Time.
|
||||
|
||||
. ../common/lib.sh
|
||||
|
||||
POOL_NAME="sanoid-test-2"
|
||||
POOL_TARGET="" # root
|
||||
RESULT="/tmp/sanoid_test_result"
|
||||
RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c"
|
||||
|
||||
# UTC timestamp of start and end
|
||||
START="1509141600"
|
||||
END="1509400800"
|
||||
|
||||
# prepare
|
||||
setup
|
||||
checkEnvironment
|
||||
disableTimeSync
|
||||
|
||||
# set timezone
|
||||
ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
|
||||
|
||||
timestamp=$START
|
||||
|
||||
mkdir -p "${POOL_TARGET}"
|
||||
truncate -s 512M "${POOL_TARGET}"/zpool2.img
|
||||
|
||||
zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool2.img
|
||||
|
||||
function cleanUp {
|
||||
zpool export "${POOL_NAME}"
|
||||
}
|
||||
|
||||
# export pool in any case
|
||||
trap cleanUp EXIT
|
||||
|
||||
while [ $timestamp -le $END ]; do
|
||||
date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose
|
||||
timestamp=$((timestamp+900))
|
||||
done
|
||||
|
||||
saveSnapshotList "${POOL_NAME}" "${RESULT}"
|
||||
|
||||
# hourly daily monthly
|
||||
verifySnapshotList "${RESULT}" 73 3 1 "${RESULT_CHECKSUM}"
|
||||
|
||||
# one more hour because of DST
|
|
@ -0,0 +1,10 @@
|
|||
[sanoid-test-2]
|
||||
use_template = production
|
||||
|
||||
[template_production]
|
||||
hourly = 36
|
||||
daily = 30
|
||||
monthly = 3
|
||||
yearly = 0
|
||||
autosnap = yes
|
||||
autoprune = no
|
|
@ -0,0 +1,107 @@
|
|||
#!/bin/bash
|
||||
|
||||
function setup {
|
||||
export LANG=C
|
||||
export LANGUAGE=C
|
||||
export LC_ALL=C
|
||||
|
||||
export SANOID="../../sanoid"
|
||||
|
||||
# make sure that there is no cache file
|
||||
rm -f /var/cache/sanoidsnapshots.txt
|
||||
|
||||
# install needed sanoid configuration files
|
||||
[ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf
|
||||
cp ../../sanoid.defaults.conf /etc/sanoid/sanoid.defaults.conf
|
||||
}
|
||||
|
||||
function checkEnvironment {
|
||||
ASK=1
|
||||
|
||||
which systemd-detect-virt > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
systemd-detect-virt --vm > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
# we are in a vm
|
||||
ASK=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $ASK -eq 1 ]; then
|
||||
set +x
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo "you should be running this test in a"
|
||||
echo "dedicated vm, as it will mess with your system!"
|
||||
echo "Are you sure you wan't to continue? (y)"
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
set -x
|
||||
|
||||
read -n 1 c
|
||||
if [ "$c" != "y" ]; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function disableTimeSync {
|
||||
# disable ntp sync
|
||||
which timedatectl > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
timedatectl set-ntp 0
|
||||
fi
|
||||
}
|
||||
|
||||
function saveSnapshotList {
|
||||
POOL_NAME="$1"
|
||||
RESULT="$2"
|
||||
|
||||
zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}"
|
||||
|
||||
# clear the seconds for comparing
|
||||
sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}"
|
||||
}
|
||||
|
||||
function verifySnapshotList {
|
||||
RESULT="$1"
|
||||
HOURLY_COUNT=$2
|
||||
DAILY_COUNT=$3
|
||||
MONTHLY_COUNT=$4
|
||||
CHECKSUM="$5"
|
||||
|
||||
failed=0
|
||||
message=""
|
||||
|
||||
hourly_count=$(grep -c "autosnap_.*_hourly" < "${RESULT}")
|
||||
daily_count=$(grep -c "autosnap_.*_daily" < "${RESULT}")
|
||||
monthly_count=$(grep -c "autosnap_.*_monthly" < "${RESULT}")
|
||||
|
||||
if [ "${hourly_count}" -ne "${HOURLY_COUNT}" ]; then
|
||||
failed=1
|
||||
message="${message}hourly snapshot count is wrong: ${hourly_count}\n"
|
||||
fi
|
||||
|
||||
if [ "${daily_count}" -ne "${DAILY_COUNT}" ]; then
|
||||
failed=1
|
||||
message="${message}daily snapshot count is wrong: ${daily_count}\n"
|
||||
fi
|
||||
|
||||
if [ "${monthly_count}" -ne "${MONTHLY_COUNT}" ]; then
|
||||
failed=1
|
||||
message="${message}monthly snapshot count is wrong: ${monthly_count}\n"
|
||||
fi
|
||||
|
||||
checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1)
|
||||
if [ "${checksum}" != "${CHECKSUM}" ]; then
|
||||
failed=1
|
||||
message="${message}result checksum mismatch\n"
|
||||
fi
|
||||
|
||||
if [ "${failed}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "TEST FAILED:" >&2
|
||||
echo -n -e "${message}" >&2
|
||||
|
||||
exit 1
|
||||
}
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
# run's all the available tests
|
||||
|
||||
for test in */; do
|
||||
if [ ! -x "${test}/run.sh" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
testName="${test%/}"
|
||||
|
||||
LOGFILE=/tmp/sanoid_test_run_"${testName}".log
|
||||
|
||||
pushd . > /dev/null
|
||||
|
||||
echo -n "Running test ${testName} ... "
|
||||
cd "${test}"
|
||||
echo | bash run.sh > "${LOGFILE}" 2>&1
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "[PASS]"
|
||||
else
|
||||
echo "[FAILED] (see ${LOGFILE})"
|
||||
fi
|
||||
|
||||
popd > /dev/null
|
||||
done
|
Loading…
Reference in New Issue