diff --git a/INSTALL b/INSTALL index 33b510d..f0de17b 100644 --- a/INSTALL +++ b/INSTALL @@ -8,7 +8,7 @@ default for SSH transport since v1.4.6. Syncoid runs will fail if one of them is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer -On CentOS: yum install lzo pv mbuffer lzop +On CentOS: yum install lzo pv mbuffer lzop perl-Data-Dumper On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than diff --git a/README.md b/README.md index 632a0af..4e1f651 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: ``` -* * * * * /usr/local/bin/sanoid --cron +* * * * * TZ=UTC /usr/local/bin/sanoid --cron ``` +`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times + And its /etc/sanoid/sanoid.conf might look something like this: ``` @@ -62,6 +64,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This option is designed to be run by a Nagios monitoring system. It reports on the health of the zpool your filesystems are on. It only monitors filesystems that are configured in the sanoid.conf file. ++ --monitor-capacity + + This option is designed to be run by a Nagios monitoring system. It reports on the capacity of the zpool your filesystems are on. It only monitors pools that are configured in the sanoid.conf file. + + --force-update This clears out sanoid's zfs snapshot listing cache. This is normally not needed. @@ -130,6 +136,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This will also transfer child datasets. ++ --skip-parent + + This will skip the syncing of the parent dataset. Does nothing without '--recursive' option. + + --compress Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. @@ -154,6 +164,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots. ++ --exclude=REGEX + + The given regular expression will be matched against all datasets which would be synced by this run and excludes them. This argument can be specified multiple times. + + --no-resume This argument tells syncoid to not use resumeable zfs send/receive streams. diff --git a/packages/debian/changelog b/packages/debian/changelog index ab530b0..2bcf423 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,18 @@ +sanoid (1.4.18) unstable; urgency=medium + + implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, + thank you @phreaker0! + + -- Jim Salter Wed, 25 Apr 2018 16:24:00 -0400 + +sanoid (1.4.17) unstable; urgency=medium + + changed die to warn when unexpectedly unable to remove a snapshot - this + allows sanoid to continue taking/removing other snapshots not affected by + whatever lock prevented the first from being taken or removed + + -- Jim Salter Wed, 8 Nov 2017 15:25:00 -0400 + sanoid (1.4.16) unstable; urgency=medium * merged @hrast01's extended fix to support -o option1=val,option2=val passthrough to SSH. merged @JakobR's diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index b54c586..2d01bbf 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -5,5 +5,6 @@ After=zfs.target ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] +Environment=TZ=UTC Type=oneshot ExecStart=/usr/sbin/sanoid --cron diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index ab299a5..3a9412f 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -58,6 +58,7 @@ Requires=zfs.target After=zfs.target [Service] +Environment=TZ=UTC Type=oneshot ExecStart=%{_sbindir}/sanoid --cron EOF diff --git a/sanoid b/sanoid index b6dc9fe..485ee08 100755 --- a/sanoid +++ b/sanoid @@ -18,7 +18,8 @@ use Time::Local; # to parse dates in reverse my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", "monitor-health", "force-update", "configdir=s", - "monitor-snapshots", "take-snapshots", "prune-snapshots" + "monitor-snapshots", "take-snapshots", "prune-snapshots", + "monitor-capacity" ) or pod2usage(2); # If only config directory (or nothing) has been specified, default to --cron --verbose @@ -39,8 +40,10 @@ my %config = init($conf_file,$default_conf_file); # if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL my $forcecacheupdate = 0; +my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); +my %pruned; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -52,6 +55,7 @@ my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath ); if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); } if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); } if ($args{'monitor-health'}) { monitor_health(@params); } +if ($args{'monitor-capacity'}) { monitor_capacity(@params); } if ($args{'force-update'}) { my $snaps = getsnaps( \%config, $cacheTTL, 1 ); } if ($args{'cron'}) { @@ -174,6 +178,61 @@ sub monitor_snapshots { exit $errorlevel; } + +#################################################################################### +#################################################################################### +#################################################################################### + +sub monitor_capacity { + my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; + my %pools; + my @messages; + my $errlevel=0; + + # build pool list with corresponding capacity limits + foreach my $section (keys %config) { + my @pool = split ('/',$section); + + if (scalar @pool == 1 || !defined($pools{$pool[0]}) ) { + my %capacitylimits; + + if (!check_capacity_limit($config{$section}{'capacity_warn'})) { + die "ERROR: invalid zpool capacity warning limit!\n"; + } + + if ($config{$section}{'capacity_warn'} != 0) { + $capacitylimits{'warn'} = $config{$section}{'capacity_warn'}; + } + + if (!check_capacity_limit($config{$section}{'capacity_crit'})) { + die "ERROR: invalid zpool capacity critical limit!\n"; + } + + if ($config{$section}{'capacity_crit'} != 0) { + $capacitylimits{'crit'} = $config{$section}{'capacity_crit'}; + } + + if (%capacitylimits) { + $pools{$pool[0]} = \%capacitylimits; + } + } + } + + foreach my $pool (keys %pools) { + my $capacitylimitsref = $pools{$pool}; + + my ($exitcode, $msg) = check_zpool_capacity($pool,\%$capacitylimitsref); + if ($exitcode > $errlevel) { $errlevel = $exitcode; } + chomp $msg; + push (@messages, $msg); + } + + my @warninglevels = ('','*** WARNING *** ','*** CRITICAL *** '); + my $message = $warninglevels[$errlevel] . join (', ',@messages); + print "$message\n"; + exit $errlevel; +} + #################################################################################### #################################################################################### #################################################################################### @@ -235,23 +294,30 @@ sub prune_snapshots { foreach my $snap( @prunesnaps ){ if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (iszfsbusy($path)) { - print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; + if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } } else { - if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; } + if (! $args{'readonly'}) { + if (system($zfs, "destroy", $snap) == 0) { + $pruned{$snap} = 1; + } else { + warn "could not remove $snap : $?"; + } + } } } removelock('sanoid_pruning'); - $forcecacheupdate = 1; - %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); + removecachedsnapshots(0); } else { - print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning - valid pruning lock held by other sanoid process.\n"; } } } } } } - + # if there were any deferred cache updates, + # do them now and wait if necessary + removecachedsnapshots(1); } # end prune_snapshots @@ -268,6 +334,19 @@ sub take_snapshots { my @newsnaps; + # get utc timestamp of the current day for DST check + my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'}); + my ($isdst) = (localtime($daystartUtc))[8]; + my $dstOffset = 0; + + if ($isdst ne $datestamp{'isdst'}) { + # current dst is different then at the beginning og the day + if ($isdst) { + # DST ended in the current day + $dstOffset = 60*60; + } + } + if ($args{'verbose'}) { print "INFO: taking snapshots...\n"; } foreach my $section (keys %config) { if ($section =~ /^template/) { next; } @@ -291,6 +370,9 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; + # to avoid duplicates with DST + my $dateSuffix = ""; + if ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; @@ -299,6 +381,13 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); + + if ($dstOffset ne 0) { + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + # DST ended, avoid duplicates + $dateSuffix = "_y"; + } if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's } elsif ($type eq 'daily') { push @preferredtime,0; # try to hit 0 seconds @@ -308,7 +397,29 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's + + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + + # check if the planned time has different DST flag than the current + my ($isdst) = (localtime($lastpreferred))[8]; + if ($isdst ne $datestamp{'isdst'}) { + if (!$isdst) { + # correct DST difference + $lastpreferred -= 60*60; + } + } + + if ($lastpreferred > time()) { + $lastpreferred -= 60*60*24; + + if ($dstOffset ne 0) { + # because we are going back one day + # the DST difference has to be accounted + # for in reverse now + $lastpreferred -= 2*$dstOffset; + } + } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'monthly_min'}; @@ -336,7 +447,7 @@ sub take_snapshots { # update to most current possible datestamp %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } @@ -484,7 +595,6 @@ sub getsnaps { my ($config, $cacheTTL, $forcecacheupdate) = @_; - my $cache = '/var/cache/sanoidsnapshots.txt'; my @rawsnaps; my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cache); @@ -521,7 +631,7 @@ sub getsnaps { } foreach my $snap (@rawsnaps) { - my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\s*creation\s*(\d*)/); + my ($fs,$snapname,$snapdate) = ($snap =~ m/(.*)\@(.*ly)\t*creation\t*(\d*)/); # avoid pissing off use warnings if (defined $snapname) { @@ -866,6 +976,11 @@ sub check_zpool() { ## other cases my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + if (!defined($sta)) { + # cache and logs are special and don't have a status + next; + } + ## pool online, not degraded thanks to dead/corrupted disk if ($state eq "OK" && $sta eq "UNAVAIL") { $state="WARNING"; @@ -900,6 +1015,74 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() +sub check_capacity_limit() { + my $value = shift; + + if (!defined($value) || $value !~ /^\d+\z/) { + return undef; + } + + if ($value < 0 || $value > 100) { + return undef; + } + + return 1 +} + +sub check_zpool_capacity() { + my %ERRORS=('DEPENDENT'=>4,'UNKNOWN'=>3,'OK'=>0,'WARNING'=>1,'CRITICAL'=>2); + my $state="UNKNOWN"; + my $msg="FAILURE"; + + my $pool=shift; + my $capacitylimitsref=shift; + my %capacitylimits=%$capacitylimitsref; + + my $statcommand="/sbin/zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + print ("$state '$statcommand' command returns no result!\n"); + exit $ERRORS{$state}; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + $state = "CRITICAL"; + $msg = sprintf "ZPOOL {%s} does not exist and/or is not responding!\n", $pool; + print $state, " ", $msg; + exit ($ERRORS{$state}); + } + + $state="OK"; + + # check capacity + my $capn = $cap; + $capn =~ s/\D//g; + + if (defined($capacitylimits{"warn"})) { + if ($capn >= $capacitylimits{"warn"}) { + $state = "WARNING"; + } + } + + if (defined($capacitylimits{"crit"})) { + if ($capn >= $capacitylimits{"crit"}) { + $state = "CRITICAL"; + } + } + + $msg = sprintf "ZPOOL %s : %s\n", $pool, $cap; + $msg = "$state $msg"; + return ($ERRORS{$state},$msg); +} # end check_zpool_capacity() + ###################################################################################################### ###################################################################################################### ###################################################################################################### @@ -930,13 +1113,22 @@ sub checklock { # no lockfile return 1; } + # make sure lockfile contains something + if ( -z $lockfile) { + # zero size lockfile, something is wrong + die "ERROR: something is wrong! $lockfile is empty\n"; + } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if # there's still a process running with that pid and with the same mutex. - open FH, "< $lockfile"; + open FH, "< $lockfile" or die "ERROR: unable to open $lockfile"; my @lock = ; close FH; + # if we didn't get exactly 2 items from the lock file there is a problem + if (scalar(@lock) != 2) { + die "ERROR: $lockfile is invalid.\n" + } my $lockmutex = pop(@lock); my $lockpid = pop(@lock); @@ -948,7 +1140,6 @@ sub checklock { # we own the lockfile. no need to check any further. return 2; } - open PL, "$pscmd -p $lockpid -o args= |"; my @processlist = ; close PL; @@ -1056,6 +1247,55 @@ sub getchilddatasets { return @children; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub removecachedsnapshots { + my $wait = shift; + + if (not %pruned) { + return; + } + + my $unlocked = checklock('sanoid_cacheupdate'); + + if ($wait != 1 && not $unlocked) { + if ($args{'verbose'}) { print "INFO: deferring cache update (snapshot removal) - valid cache update lock held by another sanoid process.\n"; } + return; + } + + # wait until we can get a lock to do our cache changes + while (not $unlocked) { + if ($args{'verbose'}) { print "INFO: waiting for cache update lock held by another sanoid process.\n"; } + sleep(10); + $unlocked = checklock('sanoid_cacheupdate'); + } + + writelock('sanoid_cacheupdate'); + + if ($args{'verbose'}) { + print "INFO: removing destroyed snapshots from cache.\n"; + } + open FH, "< $cache"; + my @rawsnaps = ; + close FH; + + open FH, "> $cache" or die 'Could not write to $cache!\n'; + foreach my $snapline ( @rawsnaps ) { + my @columns = split("\t", $snapline); + my $snap = $columns[0]; + print FH $snapline unless ( exists($pruned{$snap}) ); + } + close FH; + + removelock('sanoid_cacheupdate'); + %snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate); + + # clear hash + undef %pruned; +} + __END__ =head1 NAME @@ -1079,6 +1319,7 @@ Options: --force-update Clears out sanoid's zfs snapshot cache --monitor-health Reports on zpool "health", in a Nagios compatible format + --monitor-capacity Reports on zpool capacity, in a Nagios compatible format --monitor-snapshots Reports on snapshot "health", in a Nagios compatible format --take-snapshots Creates snapshots as specified in sanoid.conf --prune-snapshots Purges expired snapshots as specified in sanoid.conf diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..d86cc47 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -70,3 +70,7 @@ monthly_warn = 32 monthly_crit = 35 yearly_warn = 0 yearly_crit = 0 + +# default limits for capacity checks (if set to 0, limit will not be checked) +capacity_warn = 80 +capacity_crit = 95 diff --git a/syncoid b/syncoid index dcb9e2f..d25d9de 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -152,6 +152,25 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'skip-parent'}) { + # parent dataset is the first element + shift @children; + } + + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + for my $i ( 0 .. $#children ) { + if ($children[$i] =~ /$_/) { + if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } + undef $children[$i] + } + } + + @children = grep{ defined }@children; + } + } + return @children; } @@ -434,6 +453,18 @@ sub compressargset { decomrawcmd => '/usr/bin/pigz', decomargs => '-dc', }, + 'zstd-fast' => { + rawcmd => '/usr/bin/zstd', + args => '-3', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, + 'zstd-slow' => { + rawcmd => '/usr/bin/zstd', + args => '-19', + decomrawcmd => '/usr/bin/zstd', + decomargs => '-dc', + }, 'lzo' => { rawcmd => '/usr/bin/lzop', args => '', @@ -444,7 +475,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -675,7 +706,7 @@ sub getzfsvalue { open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; my $value = ; close FH; - my @values = split(/\s/,$value); + my @values = split(/\t/,$value); $value = $values[2]; return $value; } @@ -996,7 +1027,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*guid/) { chomp $line; my $guid = $line; - $guid =~ s/^.*\sguid\s*(\d*).*/$1/; + $guid =~ s/^.*\tguid\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tguid.*$/$1/; $snaps{$type}{$snap}{'guid'}=$guid; @@ -1008,7 +1039,7 @@ sub getsnaps() { if ($line =~ /\Q$fs\E\@.*creation/) { chomp $line; my $creation = $line; - $creation =~ s/^.*\screation\s*(\d*).*/$1/; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; my $snap = $line; $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; $snaps{$type}{$snap}{'creation'}=$creation; @@ -1067,9 +1098,9 @@ sub getsendsize { # the output format is different in case of # a resumed receive if (defined($receivetoken)) { - $sendsize =~ s/.*\s([0-9]+)$/$1/; + $sendsize =~ s/.*\t([0-9]+)$/$1/; } else { - $sendsize =~ s/^size\s*//; + $sendsize =~ s/^size\t*//; } chomp $sendsize; @@ -1149,10 +1180,12 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets + --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing + --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sshkey=FILE Specifies a ssh public key to use to connect --sshport=PORT Connects to remote on a particular port @@ -1160,7 +1193,7 @@ Options: --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times --help Prints this helptext - --verbose Prints the version number + --version Prints the version number --debug Prints out a lot of additional information during a syncoid run --monitor-version Currently does nothing --quiet Suppresses non-error output diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh new file mode 100755 index 0000000..7cec813 --- /dev/null +++ b/tests/1_one_year/run.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x + +# this test will take hourly, daily and monthly snapshots +# for the whole year of 2017 in the timezone Europe/Vienna +# sanoid is run hourly and no snapshots are pruned + +. ../common/lib.sh + +POOL_NAME="sanoid-test-1" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" + +# UTC timestamp of start and end +START="1483225200" +END="1514761199" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 5120M "${POOL_TARGET}"/zpool.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+3600)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" + +# hourly count should be 8760 but one hour get's lost because of DST + +# daily count should be 365 but one additional daily is taken +# because the DST change leads to a day with 25 hours +# which will trigger an additional daily snapshot diff --git a/tests/1_one_year/sanoid.conf b/tests/1_one_year/sanoid.conf new file mode 100644 index 0000000..f5692f0 --- /dev/null +++ b/tests/1_one_year/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-1] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh new file mode 100755 index 0000000..eba21ed --- /dev/null +++ b/tests/2_dst_handling/run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -x + +# this test will check the behaviour arround a date where DST ends +# with hourly, daily and monthly snapshots checked in a 15 minute interval + +# Daylight saving time 2017 in Europe/Vienna began at 02:00 on Sunday, 26 March +# and ended at 03:00 on Sunday, 29 October. All times are in +# Central European Time. + +. ../common/lib.sh + +POOL_NAME="sanoid-test-2" +POOL_TARGET="" # root +RESULT="/tmp/sanoid_test_result" +RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c" + +# UTC timestamp of start and end +START="1509141600" +END="1509400800" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 512M "${POOL_TARGET}"/zpool2.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool2.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +while [ $timestamp -le $END ]; do + date --utc --set @$timestamp; date; "${SANOID}" --cron --verbose + timestamp=$((timestamp+900)) +done + +saveSnapshotList "${POOL_NAME}" "${RESULT}" + +# hourly daily monthly +verifySnapshotList "${RESULT}" 73 3 1 "${RESULT_CHECKSUM}" + +# one more hour because of DST diff --git a/tests/2_dst_handling/sanoid.conf b/tests/2_dst_handling/sanoid.conf new file mode 100644 index 0000000..7ded3f8 --- /dev/null +++ b/tests/2_dst_handling/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-2] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no diff --git a/tests/common/lib.sh b/tests/common/lib.sh new file mode 100644 index 0000000..78f128b --- /dev/null +++ b/tests/common/lib.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +function setup { + export LANG=C + export LANGUAGE=C + export LC_ALL=C + + export SANOID="../../sanoid" + + # make sure that there is no cache file + rm -f /var/cache/sanoidsnapshots.txt + + # install needed sanoid configuration files + [ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf + cp ../../sanoid.defaults.conf /etc/sanoid/sanoid.defaults.conf +} + +function checkEnvironment { + ASK=1 + + which systemd-detect-virt > /dev/null + if [ $? -eq 0 ]; then + systemd-detect-virt --vm > /dev/null + if [ $? -eq 0 ]; then + # we are in a vm + ASK=0 + fi + fi + + if [ $ASK -eq 1 ]; then + set +x + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "you should be running this test in a" + echo "dedicated vm, as it will mess with your system!" + echo "Are you sure you wan't to continue? (y)" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + read -n 1 c + if [ "$c" != "y" ]; then + exit 1 + fi + fi +} + +function disableTimeSync { + # disable ntp sync + which timedatectl > /dev/null + if [ $? -eq 0 ]; then + timedatectl set-ntp 0 + fi +} + +function saveSnapshotList { + POOL_NAME="$1" + RESULT="$2" + + zfs list -t snapshot -o name -Hr "${POOL_NAME}" | sort > "${RESULT}" + + # clear the seconds for comparing + sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" +} + +function verifySnapshotList { + RESULT="$1" + HOURLY_COUNT=$2 + DAILY_COUNT=$3 + MONTHLY_COUNT=$4 + CHECKSUM="$5" + + failed=0 + message="" + + hourly_count=$(grep -c "autosnap_.*_hourly" < "${RESULT}") + daily_count=$(grep -c "autosnap_.*_daily" < "${RESULT}") + monthly_count=$(grep -c "autosnap_.*_monthly" < "${RESULT}") + + if [ "${hourly_count}" -ne "${HOURLY_COUNT}" ]; then + failed=1 + message="${message}hourly snapshot count is wrong: ${hourly_count}\n" + fi + + if [ "${daily_count}" -ne "${DAILY_COUNT}" ]; then + failed=1 + message="${message}daily snapshot count is wrong: ${daily_count}\n" + fi + + if [ "${monthly_count}" -ne "${MONTHLY_COUNT}" ]; then + failed=1 + message="${message}monthly snapshot count is wrong: ${monthly_count}\n" + fi + + checksum=$(sha256sum "${RESULT}" | cut -d' ' -f1) + if [ "${checksum}" != "${CHECKSUM}" ]; then + failed=1 + message="${message}result checksum mismatch\n" + fi + + if [ "${failed}" -eq 0 ]; then + exit 0 + fi + + echo "TEST FAILED:" >&2 + echo -n -e "${message}" >&2 + + exit 1 +} diff --git a/tests/run-tests.sh b/tests/run-tests.sh new file mode 100755 index 0000000..a8469e9 --- /dev/null +++ b/tests/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/sanoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done