From d8613d13797db640f9cf7432f29677056bddeb43 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 7 Nov 2017 08:03:53 +0100 Subject: [PATCH 01/51] implemented frequent snapshots with configurable period Fixes #75 --- sanoid | 21 +++++++++++++++++---- sanoid.conf | 2 ++ sanoid.defaults.conf | 7 +++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -121,12 +121,13 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly','daily','hourly'); + my @types = ('yearly','monthly','daily','hourly','frequently'); foreach my $type (@types) { my $smallerperiod = 0; # we need to set the period length in seconds first - if ($type eq 'hourly') { $smallerperiod = 60; } + if ($type eq 'frequently') { $smallerperiod = 1; } + elsif ($type eq 'hourly') { $smallerperiod = 60; } elsif ($type eq 'daily') { $smallerperiod = 60*60; } elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; } elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; } @@ -200,7 +201,8 @@ sub prune_snapshots { unless ($type =~ /ly$/) { next; } # we need to set the period length in seconds first - if ($type eq 'hourly') { $period = 60*60; } + if ($type eq 'frequently') { $period = 60 * $config{$section}{'frequent_period'}; } + elsif ($type eq 'hourly') { $period = 60*60; } elsif ($type eq 'daily') { $period = 60*60*24; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } @@ -291,7 +293,18 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; - if ($type eq 'hourly') { + if ($type eq 'frequently') { + my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'}); + + push @preferredtime,0; # try to hit 0 seconds + push @preferredtime,$frequentslice * $config{$section}{'frequent_period'}; + push @preferredtime,$datestamp{'hour'}; + push @preferredtime,$datestamp{'mday'}; + push @preferredtime,($datestamp{'mon'}-1); # january is month 0 + push @preferredtime,$datestamp{'year'}; + $lastpreferred = timelocal(@preferredtime); + if ($lastpreferred > time()) { $lastpreferred -= 60 * $config{$section}{'frequent_period'}; } # preferred time is later this frequent period - so look at last frequent period + } elsif ($type eq 'hourly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'hourly_min'}; push @preferredtime,$datestamp{'hour'}; diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..b999634 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -40,6 +40,7 @@ daily = 60 [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 @@ -49,6 +50,7 @@ [template_backup] autoprune = yes + frequently = 0 hourly = 30 daily = 90 monthly = 12 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..b5e4e63 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,12 +16,17 @@ recursive = use_template = process_children_only = +# The period in minutes for frequent snapshots, +# should be in the range of 1-30 and divide an hour without remainder +frequent_period = 15 + # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. # # Otherwise, if autoprune is set, we will prune any snapshots of that type which are older # than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days. autoprune = yes +frequently = 0 hourly = 48 daily = 90 monthly = 6 @@ -62,6 +67,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no +frequently_warn = 2000 +frequently_crit = 8000 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 20:25:07 +0100 Subject: [PATCH 02/51] hardcoded new defaults --- sanoid | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sanoid b/sanoid index e8756db..6428cdf 100755 --- a/sanoid +++ b/sanoid @@ -553,6 +553,18 @@ sub getsnaps { #################################################################################### #################################################################################### +sub verify_option_existence { + my ($hash, $key, $default) = @_; + + if (! defined (%$hash{$key})) { + $hash->{$key} = $default; + } +} + +#################################################################################### +#################################################################################### +#################################################################################### + sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -568,6 +580,12 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # hardcoded defaults which may be missing from older default configuration file + verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); + verify_option_existence($defaults{'template_default'}, 'frequently', 0); + verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); + verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 8bd98f18008494e27c47c6d16cc1fbbaec9a33a6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:40:30 +0100 Subject: [PATCH 03/51] added more documentation for frequent snapshots --- README.md | 1 + sanoid.defaults.conf | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c4e6ba..863b697 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ And its /etc/sanoid/sanoid.conf might look something like this: ############################# [template_production] + frequently = 0 hourly = 36 daily = 30 monthly = 3 diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index b5e4e63..e7c22a8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -16,8 +16,16 @@ recursive = use_template = process_children_only = -# The period in minutes for frequent snapshots, -# should be in the range of 1-30 and divide an hour without remainder +# for snapshots shorter than one hour, the period duration must be defined +# in minutes. Because they are executed within a full hour, the selected +# value should divide 60 minutes without remainder so taken snapshots +# are apart in equal intervals. Values larger than 59 aren't practical +# as only one snapshot will be taken on each full hour in this case. +# examples: +# frequent_period = 15 -> four snapshot each hour 15 minutes apart +# frequent_period = 5 -> twelve snapshots each hour 5 minutes apart +# frequent_period = 45 -> two snapshots each hour with different time gaps +# between them: 45 minutes and 15 minutes in this case frequent_period = 15 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately From bc2f8bd4e9b150294bad5c3150a682a3e857e3e0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 21:48:37 +0100 Subject: [PATCH 04/51] Revert "hardcoded new defaults" This reverts commit c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb. --- sanoid | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sanoid b/sanoid index 6428cdf..e8756db 100755 --- a/sanoid +++ b/sanoid @@ -553,18 +553,6 @@ sub getsnaps { #################################################################################### #################################################################################### -sub verify_option_existence { - my ($hash, $key, $default) = @_; - - if (! defined (%$hash{$key})) { - $hash->{$key} = $default; - } -} - -#################################################################################### -#################################################################################### -#################################################################################### - sub init { my ($conf_file, $default_conf_file) = @_; my %config; @@ -580,12 +568,6 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); - # hardcoded defaults which may be missing from older default configuration file - verify_option_existence($defaults{'template_default'}, 'frequent_period', 15); - verify_option_existence($defaults{'template_default'}, 'frequently', 0); - verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000); - verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000); - foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. From 293d83bdaa6155e765432a9a145a786d72afed9e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 8 Nov 2017 22:08:02 +0100 Subject: [PATCH 05/51] versioning and compatibility check for default configuration file --- sanoid | 12 ++++++++++++ sanoid.defaults.conf | 2 ++ 2 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index e8756db..452f535 100755 --- a/sanoid +++ b/sanoid @@ -5,6 +5,7 @@ # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. $::VERSION = '1.4.17'; +my $MINIMUM_DEFAULTS_VERSION = 2; use strict; use warnings; @@ -568,6 +569,17 @@ sub init { my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); + # check if default configuration file is up to date + my $defaults_version = 1; + if (defined $defaults{'version'}{'version'}) { + $defaults_version = $defaults{'version'}{'version'}; + delete $defaults{'version'}; + } + + if ($defaults_version < $MINIMUM_DEFAULTS_VERSION) { + die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION"; + } + foreach my $section (keys %ini) { # first up - die with honor if unknown parameters are set in any modules or templates by the user. diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index e7c22a8..06fc714 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -5,6 +5,8 @@ # # # you have been warned. # ################################################################################### +[version] +version = 2 [template_default] From 00b920682913641732981c233acd9262671719af Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 09:44:40 +0100 Subject: [PATCH 06/51] implemented weekly period --- sanoid | 26 +++++++++++++++++++++++--- sanoid.defaults.conf | 11 +++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sanoid b/sanoid index d6e58ce..9b6c815 100755 --- a/sanoid +++ b/sanoid @@ -121,15 +121,16 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly','daily','hourly'); + my @types = ('yearly','monthly', 'weekly', 'daily','hourly'); foreach my $type (@types) { my $smallerperiod = 0; # we need to set the period length in seconds first if ($type eq 'hourly') { $smallerperiod = 60; } elsif ($type eq 'daily') { $smallerperiod = 60*60; } - elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; } - elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; } + elsif ($type eq 'weekly') { $smallerperiod = 60*60*24; } + elsif ($type eq 'monthly') { $smallerperiod = 60*60*24*7; } + elsif ($type eq 'yearly') { $smallerperiod = 60*60*24*31; } my $typewarn = $type . '_warn'; my $typecrit = $type . '_crit'; @@ -202,6 +203,7 @@ sub prune_snapshots { # we need to set the period length in seconds first if ($type eq 'hourly') { $period = 60*60; } elsif ($type eq 'daily') { $period = 60*60*24; } + elsif ($type eq 'weekly') { $period = 60*60*24*7; } elsif ($type eq 'monthly') { $period = 60*60*24*31; } elsif ($type eq 'yearly') { $period = 60*60*24*365.25; } @@ -309,6 +311,24 @@ sub take_snapshots { push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's + } elsif ($type eq 'weekly') { + # calculate offset in seconds for the desired weekday + my $offset; + if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) { + $offset += 6; + } + $offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'}; + $offset *= 60*60*24; # full day + + push @preferredtime,0; # try to hit 0 seconds + push @preferredtime,$config{$section}{'weekly_min'}; + push @preferredtime,$config{$section}{'weekly_hour'}; + push @preferredtime,$datestamp{'mday'}; + push @preferredtime,($datestamp{'mon'}-1); # january is month 0 + push @preferredtime,$datestamp{'year'}; + $lastpreferred = timelocal(@preferredtime); + $lastpreferred -= $offset; + if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds push @preferredtime,$config{$section}{'monthly_min'}; diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 35c804d..7187104 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -24,6 +24,7 @@ process_children_only = autoprune = yes hourly = 48 daily = 90 +weekly = 0 monthly = 6 yearly = 0 min_percent_free = 10 @@ -40,6 +41,10 @@ hourly_min = 0 # daily - at 23:59 (most people expect a daily to contain everything done DURING that day) daily_hour = 23 daily_min = 59 +# weekly -at 23:30 each Monday +weekly_wday = 1 +weekly_hour = 23 +weekly_min = 30 # monthly - immediately at the beginning of the month (ie 00:00 of day 1) monthly_mday = 1 monthly_hour = 0 @@ -66,7 +71,9 @@ hourly_warn = 90 hourly_crit = 360 daily_warn = 28 daily_crit = 32 -monthly_warn = 32 -monthly_crit = 35 +weekly_warn = 7 +weekly_crit = 10 +monthly_warn = 5 +monthly_crit = 6 yearly_warn = 0 yearly_crit = 0 From 4649704046cee190c8482e00e79b9b8d408829e4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 17:41:52 +0100 Subject: [PATCH 07/51] codestyle fix and disable monitoring of weekly snapshots --- sanoid | 2 +- sanoid.defaults.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 9b6c815..05c4def 100755 --- a/sanoid +++ b/sanoid @@ -121,7 +121,7 @@ sub monitor_snapshots { my $path = $config{$section}{'path'}; push @paths, $path; - my @types = ('yearly','monthly', 'weekly', 'daily','hourly'); + my @types = ('yearly','monthly','weekly','daily','hourly'); foreach my $type (@types) { my $smallerperiod = 0; diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 7187104..ff79ebb 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -71,8 +71,8 @@ hourly_warn = 90 hourly_crit = 360 daily_warn = 28 daily_crit = 32 -weekly_warn = 7 -weekly_crit = 10 +weekly_warn = 0 +weekly_crit = 0 monthly_warn = 5 monthly_crit = 6 yearly_warn = 0 From ac16b2128e6863a8af619d0bf9282bf92f6dfbc9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 9 Nov 2017 17:44:04 +0100 Subject: [PATCH 08/51] disable monitoring of frequent snapshots --- sanoid.defaults.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 06fc714..a521401 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -77,8 +77,8 @@ yearly_min = 0 monitor = yes monitor_dont_warn = no monitor_dont_crit = no -frequently_warn = 2000 -frequently_crit = 8000 +frequently_warn = 0 +frequently_crit = 0 hourly_warn = 90 hourly_crit = 360 daily_warn = 28 From 2a3d91e4465ea35ce68548915aeaf4eb1ec75745 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 13 Dec 2017 00:44:48 +0100 Subject: [PATCH 09/51] fixed weekly snapshot interval --- sanoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 05c4def..792497a 100755 --- a/sanoid +++ b/sanoid @@ -313,9 +313,9 @@ sub take_snapshots { if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'weekly') { # calculate offset in seconds for the desired weekday - my $offset; + my $offset = 0; if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) { - $offset += 6; + $offset += 7; } $offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'}; $offset *= 60*60*24; # full day @@ -327,7 +327,7 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - $lastpreferred -= $offset; + $lastpreferred += $offset; if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's } elsif ($type eq 'monthly') { push @preferredtime,0; # try to hit 0 seconds From 99dcf7f3406b64f7f6ae261b41666fd2bbbaeec9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:52:36 +0200 Subject: [PATCH 10/51] implemented support for zfs bookmarks, if no matching snapshots are found bookmarks are tried as fallback, both stream and no-stream cases are supported --- syncoid | 180 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 148 insertions(+), 32 deletions(-) diff --git a/syncoid b/syncoid index 6c569b2..3cbff6b 100755 --- a/syncoid +++ b/syncoid @@ -370,11 +370,48 @@ sub syncdataset { my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps); + my $bookmark = 0; + my $bookmarkcreation = 0; + + my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps); if (! $matchingsnap) { - # no matching snapshot; we whined piteously already, but let's go ahead and return false - # now in case more child datasets need replication. - return 0; + # no matching snapshots, check for bookmarks as fallback + my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot); + + # check for matching guid of source bookmark and target snapshot (oldest first) + foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) { + my $guid = $snaps{'target'}{$snap}{'guid'}; + + if (defined $bookmarks{$guid}) { + # found a match + $bookmark = $bookmarks{$guid}{'name'}; + $bookmarkcreation = $bookmarks{$guid}{'creation'}; + $matchingsnap = $snap; + last; + } + } + + if (! $bookmark) { + # if we got this far, we failed to find a matching snapshot/bookmark. + + print "\n"; + print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; + print " Replication to target would require destroying existing\n"; + print " target. Cowardly refusing to destroy your existing target.\n\n"; + + # experience tells me we need a mollyguard for people who try to + # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... + + if ( $targetsize < (64*1024*1024) ) { + print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; + print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; + print " replication must be to a NON EXISTENT DATASET, which will\n"; + print " then be CREATED BY the initial replication process.\n\n"; + } + + # return false now in case more child datasets need replication. + return 0; + } } # make sure target is (still) not currently in receive. @@ -398,17 +435,68 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); - my $disp_pvsize = readablebytes($pvsize); - if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } - my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + my $nextsnapshot = 0; - if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } - if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + if ($bookmark) { + my $bookmarkescaped = escapeshellparam($bookmark); + + if (!defined $args{'no-stream'}) { + # if intermediate snapshots are needed we need to find the next oldest snapshot, + # do an replication to it and replicate as always from oldest to newest + # because bookmark sends doesn't support intermediates directly + foreach my $snap ( sort { $snaps{'source'}{$a}{'creation'}<=>$snaps{'source'}{$b}{'creation'} } keys %{ $snaps{'source'} }) { + if ($snaps{'source'}{$snap}{'creation'} >= $bookmarkcreation) { + $nextsnapshot = $snap; + last; + } + } + } + + # bookmark stream size can't be determined + my $pvsize = 0; + my $disp_pvsize = "UNKNOWN"; + + if ($nextsnapshot) { + my $nextsnapshotescaped = escapeshellparam($nextsnapshot); + + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + $matchingsnap = $nextsnapshot; + $matchingsnapescaped = escapeshellparam($matchingsnap); + } else { + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } + } + + # do a normal replication if bookmarks aren't used or if previous + # bookmark replication was only done to the next oldest snapshot + if (!$bookmark || $nextsnapshot) { + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior @@ -900,31 +988,15 @@ sub pruneoldsyncsnaps { } sub getmatchingsnapshot { - my ($sourcefs, $targetfs, $targetsize, $snaps) = @_; + my ($sourcefs, $targetfs, $snaps) = @_; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { - if (defined $snaps{'target'}{$snap}{'guid'}) { + if (defined $snaps{'target'}{$snap}) { if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) { return $snap; } } } - # if we got this far, we failed to find a matching snapshot. - - print "\n"; - print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; - print " Replication to target would require destroying existing\n"; - print " target. Cowardly refusing to destroy your existing target.\n\n"; - - # experience tells me we need a mollyguard for people who try to - # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... - - if ( $targetsize < (64*1024*1024) ) { - print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; - print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; - print " replication must be to a NON EXISTENT DATASET, which will\n"; - print " then be CREATED BY the initial replication process.\n\n"; - } return 0; } @@ -1049,6 +1121,50 @@ sub getsnaps() { return %snaps; } +sub getbookmarks() { + my ($rhost,$fs,$isroot,%bookmarks) = @_; + my $mysudocmd; + my $fsescaped = escapeshellparam($fs); + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |"; + if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; } + open FH, $getbookmarkcmd; + my @rawbookmarks = ; + close FH; + + # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines + # as though each were an entirely separate get command. + + my $lastguid; + + foreach my $line (@rawbookmarks) { + # only import bookmark guids, creation from the specified filesystem + if ($line =~ /\Q$fs\E\#.*guid/) { + chomp $line; + $lastguid = $line; + $lastguid =~ s/^.*\tguid\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tguid.*$/$1/; + $bookmarks{$lastguid}{'name'}=$bookmark; + } elsif ($line =~ /\Q$fs\E\#.*creation/) { + chomp $line; + my $creation = $line; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/; + $bookmarks{$lastguid}{'creation'}=$creation; + } + } + + return %bookmarks; +} sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; From a1d9e79e70548b88e4f702d3790d9f5e7c028ffc Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:58:42 +0200 Subject: [PATCH 11/51] added two tests for zfs bookmark replication --- .../run.sh | 56 +++++++++++++++++++ .../run.sh | 56 +++++++++++++++++++ tests/syncoid/run-tests.sh | 27 +++++++++ 3 files changed, 139 insertions(+) create mode 100755 tests/syncoid/1_bookmark_replication_intermediate/run.sh create mode 100755 tests/syncoid/2_bookmark_replication_no_intermediate/run.sh create mode 100755 tests/syncoid/run-tests.sh diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/1_bookmark_replication_intermediate/run.sh new file mode 100755 index 0000000..11edb04 --- /dev/null +++ b/tests/syncoid/1_bookmark_replication_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-1.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-1" +TARGET_CHECKSUM="a23564d5bb8a2babc3ac8936fd82825ad9fff9c82d4924f5924398106bbda9f0 -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh new file mode 100755 index 0000000..94ac690 --- /dev/null +++ b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-2.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-2" +TARGET_CHECKSUM="2460d4d4417793d2c7a5c72cbea4a8a584c0064bf48d8b6daa8ba55076cba66d -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/run-tests.sh b/tests/syncoid/run-tests.sh new file mode 100755 index 0000000..a9843a5 --- /dev/null +++ b/tests/syncoid/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/syncoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done From 089516c58ec43c30342a45abebd60b60b94e41a4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 12:43:46 +0200 Subject: [PATCH 12/51] implemented force-delete flag to let syncoid destroy target datasets without matching snapshots/bookmarks --- syncoid | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 3cbff6b..8554d43 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,8 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", + "force-delete") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -176,7 +177,7 @@ sub getchilddatasets { sub syncdataset { - my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my ($sourcehost, $sourcefs, $targethost, $targetfs, $skipsnapshot) = @_; my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); @@ -228,7 +229,7 @@ sub syncdataset { print "\n\n\n"; } - if (!defined $args{'no-sync-snap'}) { + if (!defined $args{'no-sync-snap'} && !defined $skipsnapshot) { # create a new syncoid snapshot on the source filesystem. $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); } else { @@ -392,6 +393,30 @@ sub syncdataset { } if (! $bookmark) { + if ($args{'force-delete'}) { + if (!$quiet) { print "Removing $targetfs because no matching snapshots were found\n"; } + + my $rcommand = ''; + my $mysudocmd = ''; + my $targetfsescaped = escapeshellparam($targetfs); + + if ($targethost ne '') { $rcommand = "$sshcmd $targethost"; } + if (!$targetisroot) { $mysudocmd = $sudocmd; } + + my $prunecmd = "$mysudocmd $zfscmd destroy -r $targetfsescaped; "; + if ($targethost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } + + my $ret = system("$rcommand $prunecmd"); + if ($ret != 0) { + warn "WARNING: $rcommand $prunecmd failed: $?"; + } else { + # redo sync and skip snapshot creation (already taken) + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, 1); + } + } + # if we got this far, we failed to find a matching snapshot/bookmark. print "\n"; From 8905c003355b27c5bd84ce4da74f834bef85df1c Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sun, 29 Jul 2018 12:45:12 +0200 Subject: [PATCH 13/51] added test for force-delete flag --- tests/syncoid/3_force_delete/run.sh | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 tests/syncoid/3_force_delete/run.sh diff --git a/tests/syncoid/3_force_delete/run.sh b/tests/syncoid/3_force_delete/run.sh new file mode 100755 index 0000000..03ad9fa --- /dev/null +++ b/tests/syncoid/3_force_delete/run.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# test replication with deletion of target if no matches are found + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-3.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-3" +TARGET_CHECKSUM="0409a2ac216e69971270817189cef7caa91f6306fad9eab1033955b7e7c6bd4c -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs create "${POOL_NAME}"/src/1 +zfs create "${POOL_NAME}"/src/2 +zfs create "${POOL_NAME}"/src/3 + +# initial replication +../../../syncoid -r --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src/2@% +zfs snapshot "${POOL_NAME}"/src/2@test +sleep 1 +../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name | sed 's/@syncoid_.*$'/@syncoid_/) +checksum=$(echo "${output}" | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 From 1cb1209c785a43bd6d162d80ea75dc36f86a99b4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 09:13:26 +0200 Subject: [PATCH 14/51] documented force-delete flag --- README.md | 4 ++++ syncoid | 1 + 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index e359886..4755571 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to not use resumeable zfs send/receive streams. ++ --force-delete + + Remove target datasets recursively (WARNING: this will also affect child datasets with matching snapshots/bookmarks), if there are no matching snapshots/bookmarks. + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index 8554d43..a6500bb 100755 --- a/syncoid +++ b/syncoid @@ -1341,3 +1341,4 @@ Options: --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available + --force-delete Remove target datasets recursively, if there are no matching snapshots/bookmarks From 020f57cb746ad7b5ed9a0fafa0a80b427cf6f3ae Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 30 Jul 2018 09:43:03 +0200 Subject: [PATCH 15/51] check for error while listing snapshots/bookmarks as this can be dangerous with force-delete --- syncoid | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index a6500bb..26307ba 100755 --- a/syncoid +++ b/syncoid @@ -1114,7 +1114,7 @@ sub getsnaps() { if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; - close FH; + close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. @@ -1158,11 +1158,21 @@ sub getbookmarks() { $fsescaped = escapeshellparam($fsescaped); } - my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |"; + my $error = 0; + my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped 2>&1 |"; if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; } open FH, $getbookmarkcmd; my @rawbookmarks = ; - close FH; + close FH or $error = 1; + + if ($error == 1) { + if ($rawbookmarks[0] =~ /invalid type/) { + # no support for zfs bookmarks, return empty hash + return %bookmarks; + } + + die "CRITICAL ERROR: bookmarks couldn't be listed for $fs (exit code $?)"; + } # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. From f51bb9db7e44a0d512c29c8ba01b0ad2265d237c Mon Sep 17 00:00:00 2001 From: Martin Schrodt Date: Wed, 15 Aug 2018 16:07:01 +0200 Subject: [PATCH 16/51] add xz compression --- syncoid | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index b62eee8..18708d2 100755 --- a/syncoid +++ b/syncoid @@ -509,6 +509,12 @@ sub compressargset { decomrawcmd => '/usr/bin/zstd', decomargs => '-dc', }, + 'xz' => { + rawcmd => '/usr/bin/xz', + args => '', + decomrawcmd => '/usr/bin/xz', + decomargs => '-d', + }, 'lzo' => { rawcmd => '/usr/bin/lzop', args => '', @@ -519,7 +525,7 @@ sub compressargset { if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'xz', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, xz, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 0deaacfc06bdb891525fb1bdc1ed34abcb91b9c1 Mon Sep 17 00:00:00 2001 From: Martin Schrodt Date: Wed, 15 Aug 2018 22:21:56 +0200 Subject: [PATCH 17/51] add xz to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b833dec..dc1162b 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, xz, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit From 8de3cdce212ae15a849768e824d929afd6bc1501 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:45:03 +0200 Subject: [PATCH 18/51] let monitor-health also check vdev member io/checksum errors --- sanoid | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) mode change 100755 => 100644 sanoid diff --git a/sanoid b/sanoid old mode 100755 new mode 100644 index 485ee08..7ae1b5b --- a/sanoid +++ b/sanoid @@ -974,7 +974,7 @@ sub check_zpool() { } ## other cases - my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + my ($dev, $sta, $read, $write, $cksum) = /^\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/; if (!defined($sta)) { # cache and logs are special and don't have a status @@ -994,8 +994,21 @@ sub check_zpool() { ## no display for verbose level 1 next if ($verbose==1); ## don't display working devices for verbose level 2 - next if ($verbose==2 && $state eq "OK"); - next if ($verbose==2 && ($sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")); + if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")) { + # check for io/checksum errors + + my @vdeverr = (); + if ($read != 0) { push @vdeverr, "read" }; + if ($write != 0) { push @vdeverr, "write" }; + if ($cksum != 0) { push @vdeverr, "cksum" }; + + if (scalar @vdeverr) { + $dmge=$dmge . "(" . $dev . ":" . join(", ", @vdeverr) . " errors) "; + if ($state eq "OK") { $state = "WARNING" }; + } + + next; + } ## show everything else if (/^\s{3}(\S+)/) { From 997487d12bf0f7d725199fb9fb9dd5cb4e484504 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 17:46:21 +0200 Subject: [PATCH 19/51] restore filemode --- sanoid | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sanoid diff --git a/sanoid b/sanoid old mode 100644 new mode 100755 From f39ed1ec49e989aa4fc7ae8c79f6329f5764cf3b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 3 Sep 2018 18:32:17 +0200 Subject: [PATCH 20/51] for remote target/source it's required to specify a user --- syncoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index b62eee8..30aef49 100755 --- a/syncoid +++ b/syncoid @@ -1218,9 +1218,9 @@ syncoid - ZFS snapshot replication tool =head1 SYNOPSIS syncoid [options]... SOURCE TARGET - or syncoid [options]... SOURCE [USER@]HOST:TARGET - or syncoid [options]... [USER@]HOST:SOURCE [TARGET] - or syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET + or syncoid [options]... SOURCE USER@HOST:TARGET + or syncoid [options]... USER@HOST:SOURCE TARGET + or syncoid [options]... USER@HOST:SOURCE USER@HOST:TARGET SOURCE Source ZFS dataset. Can be either local or remote TARGET Target ZFS dataset. Can be either local or remote From 03a074e5c286d77d049df9555cabd3d2cc71d4a2 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Sep 2018 08:13:47 +0200 Subject: [PATCH 21/51] show warning if a key is ignored from a template --- sanoid | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 485ee08..5764849 100755 --- a/sanoid +++ b/sanoid @@ -691,10 +691,12 @@ sub init { # override with values from user-defined default template, if any foreach my $key (keys %{$ini{'template_default'}}) { - if (! ($key =~ /template|recursive/)) { - if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; } - $config{$section}{$key} = $ini{'template_default'}{$key}; + if ($key =~ /template|recursive/) { + warn "ignored key '$key' from user-defined default template.\n"; + next; } + if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; } + $config{$section}{$key} = $ini{'template_default'}{$key}; } } @@ -708,10 +710,12 @@ sub init { my $template = 'template_'.$rawtemplate; foreach my $key (keys %{$ini{$template}}) { - if (! ($key =~ /template|recursive/)) { - if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; } - $config{$section}{$key} = $ini{$template}{$key}; + if ($key =~ /template|recursive/) { + warn "ignored key '$key' from '$rawtemplate' template.\n"; + next; } + if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; } + $config{$section}{$key} = $ini{$template}{$key}; } } } From 65c7be5b1d58287e0fa637c01b6b8265e350fe2a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Sep 2018 20:17:46 +0200 Subject: [PATCH 22/51] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b833dec..0354c6b 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ And its /etc/sanoid/sanoid.conf might look something like this: autoprune = yes ``` -Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set). Except in the case of data/images/win7-spice, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason. +Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set). Except in the case of data/images/win7, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason. ##### Sanoid Command Line Options From 2fe97f13ad29ed9f07000475bb0e9aa1430e95f7 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 10:07:05 +0200 Subject: [PATCH 23/51] prepare for v1.4.19 release --- CHANGELIST | 23 +++++++++++++++++++++++ VERSION | 2 +- packages/debian/changelog | 27 +++++++++++++++++++++++++++ packages/rhel/sanoid.spec | 6 ++++-- sanoid | 2 +- syncoid | 2 +- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 515d05d..3ef74ae 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,26 @@ +1.4.19 [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! diff --git a/VERSION b/VERSION index f689e8c..fd4ca57 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.18 +1.4.19 diff --git a/packages/debian/changelog b/packages/debian/changelog index 2bcf423..67cca4d 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,30 @@ +sanoid (1.4.19) unstable; urgency=medium + + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + + -- Jim Salter Wed, 05 Sep 2018 04:00:00 -0400 + sanoid (1.4.18) unstable; urgency=medium implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3a9412f..4971404 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 1.4.18 +%global version 1.4.19 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -12,7 +12,7 @@ Summary: A policy-driven snapshot management tool for ZFS file systems Group: Applications/System License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid -Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz +Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz Requires: perl, mbuffer, lzop, pv %if 0%{?_with_systemd} @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Wed Sep 05 2018 Christoph Klaffl - 1.4.19 +- Bump to 1.4.19 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 - Bump to 1.4.18 * Thu Aug 31 2017 Dominic Robinson - 1.4.14-2 diff --git a/sanoid b/sanoid index 7ae1b5b..fd68145 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.18'; +$::VERSION = '1.4.19'; use strict; use warnings; diff --git a/syncoid b/syncoid index 30aef49..8aa5205 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '1.4.18'; +$::VERSION = '1.4.19'; use strict; use warnings; From 055f26b9709aa412b2597e53c99e00c09cf5f5df Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 18:47:16 +0200 Subject: [PATCH 24/51] ignore unknown interval type to prevent perl warnings --- sanoid | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sanoid b/sanoid index 7ae1b5b..31376ad 100755 --- a/sanoid +++ b/sanoid @@ -438,6 +438,9 @@ sub take_snapshots { push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year + } else { + # unknown type + next; } # reconstruct our human-formatted most recent preferred snapshot time into an epoch time, to compare with the epoch of our most recent snapshot From d5f4d1c121e40da89fcca720ffd55332c3847b04 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:01:15 +0200 Subject: [PATCH 25/51] add lz4 compression --- README.md | 2 +- syncoid | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b833dec..ce09a37 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit diff --git a/syncoid b/syncoid index 30aef49..c42a008 100755 --- a/syncoid +++ b/syncoid @@ -515,11 +515,17 @@ sub compressargset { decomrawcmd => '/usr/bin/lzop', decomargs => '-dfc', }, + 'lz4' => { + rawcmd => '/usr/bin/lz4', + args => '', + decomrawcmd => '/usr/bin/lz4', + decomargs => '-dc', + }, ); if ($value eq 'default') { $value = $DEFAULT_COMPRESSION; - } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) { + } elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'lzo', 'default', 'none'))) { warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION"; $value = $DEFAULT_COMPRESSION; } @@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool Options: - --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. From 4af294838256fee6cee00d034bd75936992063d3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:18:21 +0200 Subject: [PATCH 26/51] fix uninitialized value warning in debug mode on initial run (no snapshots yet) --- sanoid | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..0af79a4 100755 --- a/sanoid +++ b/sanoid @@ -492,16 +492,20 @@ sub blabber { my $path = $config{$section}{'path'}; print "Filesystem $path has:\n"; print " $snapsbypath{$path}{'numsnaps'} total snapshots "; - print "(newest: "; - my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); - print "$newest hours old)\n"; + if ($snapsbypath{$path}{'numsnaps'} == 0) { + print "(no current snapshots)" + } else { + print "(newest: "; + my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60); + print "$newest hours old)\n"; - foreach my $type (keys %{ $snapsbytype{$path} }){ - print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; - print " desired: $config{$section}{$type}\n"; - print " newest: "; - my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); - print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + foreach my $type (keys %{ $snapsbytype{$path} }){ + print " $snapsbytype{$path}{$type}{'numsnaps'} $type\n"; + print " desired: $config{$section}{$type}\n"; + print " newest: "; + my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60)); + print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n"; + } } print "\n\n"; } From 253b8b467d2de2962826842052645d4439c0fd40 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 19:34:44 +0200 Subject: [PATCH 27/51] parse values for recursive key like the others booleans, previously any non empty value was considered as true --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 7ae1b5b..e5d3e05 100755 --- a/sanoid +++ b/sanoid @@ -743,7 +743,7 @@ sub init { # how 'bout some recursion? =) my @datasets; - if ($ini{$section}{'recursive'}) { + if (grep( /^$ini{$section}{'recursive'}$/, @istrue )) { @datasets = getchilddatasets($config{$section}{'path'}); foreach my $dataset(@datasets) { chomp $dataset; From 807fc53afb4d28b3742af506d8e3f9b0ee2a4337 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 5 Sep 2018 22:46:14 +0200 Subject: [PATCH 28/51] added all available syncoid/sanoid parameters to README --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index b833dec..9fcfcf6 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,13 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --readonly + + Skip creation/deletion of snapshots (Simulate). + ++ --help + + Show help message. ---------- @@ -206,6 +213,14 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup Allow sync to/from boxes running SSH on non-standard ports. ++ --sshcipher + + Instruct ssh to use a particular cipher set. + ++ --sshoption + + Passes option to ssh. This argument can be specified multiple times. + + --sshkey Use specified identity file as per ssh -i. @@ -218,6 +233,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This prints out quite alot of additional information during a sanoid run, and is normally not needed. ++ --help + + Show help message. + + --version Print the version and exit. From e7bd567acb1aed210f7d36cd9933460cab080da2 Mon Sep 17 00:00:00 2001 From: Andrew DeMaria Date: Mon, 10 Sep 2018 22:14:05 -0400 Subject: [PATCH 29/51] Do not monitor snapshots types that are set to 0. Signed-off-by: Andrew DeMaria --- sanoid | 1 + 1 file changed, 1 insertion(+) diff --git a/sanoid b/sanoid index 7ae1b5b..aa6369d 100755 --- a/sanoid +++ b/sanoid @@ -127,6 +127,7 @@ sub monitor_snapshots { my @types = ('yearly','monthly','daily','hourly'); foreach my $type (@types) { + if ($config{$section}{$type} == 0) { next; } my $smallerperiod = 0; # we need to set the period length in seconds first From e9a330f89a8ed9894df7d1a7bf6a996bcec3918f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 5 Oct 2018 16:19:55 +0200 Subject: [PATCH 30/51] adapt test as it works correctly know after the DST patch --- tests/1_one_year/run.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh index 7cec813..1cae7b4 100755 --- a/tests/1_one_year/run.sh +++ b/tests/1_one_year/run.sh @@ -10,7 +10,7 @@ set -x POOL_NAME="sanoid-test-1" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf" +RESULT_CHECKSUM="68c67161a59d0e248094a66061972f53613067c9db52ad981030f36bc081fed7" # UTC timestamp of start and end START="1483225200" @@ -46,10 +46,4 @@ done saveSnapshotList "${POOL_NAME}" "${RESULT}" # hourly daily monthly -verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}" - -# hourly count should be 8760 but one hour get's lost because of DST - -# daily count should be 365 but one additional daily is taken -# because the DST change leads to a day with 25 hours -# which will trigger an additional daily snapshot +verifySnapshotList "${RESULT}" 8760 365 12 "${RESULT_CHECKSUM}" From e83ec060fb68ad47545e24cde71e60c3d63f2752 Mon Sep 17 00:00:00 2001 From: Michael Bushey Date: Sun, 14 Oct 2018 15:10:25 -0700 Subject: [PATCH 31/51] INSTALL: Fix name p5-Config-Inifiles -> p5-Config-IniFiles --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index f0de17b..15c4896 100644 --- a/INSTALL +++ b/INSTALL @@ -30,4 +30,4 @@ strongly recommends using your distribution's repositories instead. On Ubuntu: apt install libconfig-inifiles-perl On CentOS: yum install perl-Config-IniFiles -On FreeBSD: pkg install p5-Config-Inifiles +On FreeBSD: pkg install p5-Config-IniFiles From c6ffbf5c4c05e8805a441656b253836631272221 Mon Sep 17 00:00:00 2001 From: Julien Riou Date: Sat, 13 Jan 2018 14:26:26 +0100 Subject: [PATCH 32/51] Add pre and post snapshot scripts --- sanoid | 25 ++++++++++++++++++++++++- sanoid.conf | 11 +++++++++++ sanoid.defaults.conf | 4 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 7ae1b5b..e45133b 100755 --- a/sanoid +++ b/sanoid @@ -455,6 +455,18 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { + my $dataset = (split '@', $snap)[0]; + my $presnapshotfailure = 0; + if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { + warn "WARN: pre_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + $presnapshotfailure = 1; + } + delete $ENV{'SANOID_TARGET'}; + } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { system($zfs, "snapshot", "$snap") == 0 @@ -462,6 +474,17 @@ sub take_snapshots { # make sure we don't end up with multiple snapshots with the same ctime sleep 1; } + if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { + if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } + if (system($config{$dataset}{'post_snapshot_script'}) != 0) { + warn "WARN: post_snapshot_script failed, $?"; + $config{$dataset}{'no_inconsistent_snapshot'} and next; + } + delete $ENV{'SANOID_TARGET'}; + } + } } $forcecacheupdate = 1; %snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate); @@ -661,7 +684,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','no_inconsistent_snapshot','force_post_snapshot_script'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); diff --git a/sanoid.conf b/sanoid.conf index 9b1f19d..218a492 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -67,6 +67,17 @@ daily_warn = 48 daily_crit = 60 +[template_scripts] + ### run script before snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pre_snapshot_script = /path/to/script.sh + ### run script after snapshot + ### dataset name will be supplied as an environment variable $SANOID_TARGET + post_snapshot_script = /path/to/script.sh + ### don't take an inconsistent snapshot + #no_inconsistent_snapshot = yes + ### run post_snapshot_script when pre_snapshot_script is failing + #force_post_snapshot_script = yes [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..12a8049 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,10 @@ path = recursive = use_template = process_children_only = +pre_snapshot_script = +post_snapshot_script = +no_inconsistent_snapshot = +force_post_snapshot_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 84213216ec2d98182e31c7e43ecce48cd353404a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Tue, 2 Oct 2018 00:47:25 +0200 Subject: [PATCH 33/51] Expose snapshot name through ENV variable --- sanoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sanoid b/sanoid index e45133b..73efd5e 100755 --- a/sanoid +++ b/sanoid @@ -456,9 +456,11 @@ sub take_snapshots { if ( (scalar(@newsnaps)) > 0) { foreach my $snap ( @newsnaps ) { my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { warn "WARN: pre_snapshot_script failed, $?"; @@ -466,6 +468,7 @@ sub take_snapshots { $presnapshotfailure = 1; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -477,12 +480,14 @@ sub take_snapshots { if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } if (system($config{$dataset}{'post_snapshot_script'}) != 0) { warn "WARN: post_snapshot_script failed, $?"; $config{$dataset}{'no_inconsistent_snapshot'} and next; } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; } } } From fb6608bf47a9508fb3b661a364e597b94b517e5f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 10:48:04 +0200 Subject: [PATCH 34/51] implemented timeout for pre/post script execution and made sure environment is cleaned up after script failure --- sanoid | 50 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/sanoid b/sanoid index 73efd5e..c12dff8 100755 --- a/sanoid +++ b/sanoid @@ -458,17 +458,21 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; + my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'pre_snapshot_script'}) != 0) { - warn "WARN: pre_snapshot_script failed, $?"; + my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + + if ($ret != 0) { + # warning was already thrown by runscript function $config{$dataset}{'no_inconsistent_snapshot'} and next; $presnapshotfailure = 1; } - delete $ENV{'SANOID_TARGET'}; - delete $ENV{'SANOID_SNAPNAME'}; } if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { @@ -482,10 +486,8 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - if (system($config{$dataset}{'post_snapshot_script'}) != 0) { - warn "WARN: post_snapshot_script failed, $?"; - $config{$dataset}{'no_inconsistent_snapshot'} and next; - } + runscript('post_snapshot_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } @@ -1337,6 +1339,38 @@ sub removecachedsnapshots { undef %pruned; } +#######################################################################################################################3 +#######################################################################################################################3 +#######################################################################################################################3 + +sub runscript { + my $key=shift; + my $dataset=shift; + my $timeout=shift; + + my $ret; + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + $ret = system($config{$dataset}{$key}); + alarm 0; + }; + if ($@) { + if ($@ eq "alarm\n") { + warn "WARN: $key didn't finish in the allowed time!"; + } else { + warn "CRITICAL ERROR: $@"; + } + return -1; + } else { + if ($ret != 0) { + warn "WARN: $key failed, $?"; + } + } + + return $ret; +} + __END__ =head1 NAME From 0a7fdcb232d5e75eed388f016060cd4736c6185a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Belli?= Date: Sun, 14 Oct 2018 16:28:24 +0200 Subject: [PATCH 35/51] Add pruning hooks --- sanoid | 11 +++++++++++ sanoid.conf | 2 ++ sanoid.defaults.conf | 1 + 3 files changed, 14 insertions(+) diff --git a/sanoid b/sanoid index c12dff8..866eef0 100755 --- a/sanoid +++ b/sanoid @@ -299,6 +299,17 @@ sub prune_snapshots { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { $pruned{$snap} = 1; + my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; + if ($config{$dataset}{'pruning_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; + if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } + system($config{$dataset}{'pruning_script'}) == 0 + or warn "WARN: pruning_script failed, $?"; + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + } } else { warn "could not remove $snap : $?"; } diff --git a/sanoid.conf b/sanoid.conf index 218a492..e684614 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -78,6 +78,8 @@ #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index 12a8049..d4dd19e 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -19,6 +19,7 @@ pre_snapshot_script = post_snapshot_script = no_inconsistent_snapshot = force_post_snapshot_script = +pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a7b7fe8d15adc826d78f914f962b0b6929df88cb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 11:58:25 +0200 Subject: [PATCH 36/51] let pruning script timeout so it doesn't hang sanoid --- sanoid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 866eef0..7c213e4 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,12 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { + my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - system($config{$dataset}{'pruning_script'}) == 0 - or warn "WARN: pruning_script failed, $?"; + my $ret = runscript('pruning_script',$dataset,$timeout); + delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; } From a8d5c5652a82c505b117ccd02a39962be240bd10 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 17:54:37 +0200 Subject: [PATCH 37/51] make script timeout configureable --- sanoid | 16 +++++++++------- sanoid.conf | 6 ++++-- sanoid.defaults.conf | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sanoid b/sanoid index 7c213e4..9c0e54d 100755 --- a/sanoid +++ b/sanoid @@ -302,11 +302,10 @@ sub prune_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { - my $timeout = 5; $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pruning_script',$dataset,$timeout); + my $ret = runscript('pruning_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -475,7 +474,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } - my $ret = runscript('pre_snapshot_script',$dataset,$timeout); + my $ret = runscript('pre_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -498,7 +497,7 @@ sub take_snapshots { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } - runscript('post_snapshot_script',$dataset,$timeout); + runscript('post_snapshot_script',$dataset); delete $ENV{'SANOID_TARGET'}; delete $ENV{'SANOID_SNAPNAME'}; @@ -1358,12 +1357,15 @@ sub removecachedsnapshots { sub runscript { my $key=shift; my $dataset=shift; - my $timeout=shift; + + my $timeout=$config{$dataset}{'script_timeout'}; my $ret; eval { - local $SIG{ALRM} = sub { die "alarm\n" }; - alarm $timeout; + if ($timeout gt 0) { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + } $ret = system($config{$dataset}{$key}); alarm 0; }; diff --git a/sanoid.conf b/sanoid.conf index e684614..db468e2 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -74,12 +74,14 @@ ### run script after snapshot ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh + ### dataset name will be supplied as an environment variable $SANOID_TARGET + pruning_script = /path/to/script.sh ### don't take an inconsistent snapshot #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### dataset name will be supplied as an environment variable $SANOID_TARGET - pruning_script = /path/to/script.sh + ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + script_timeout = 5 [template_ignore] autoprune = no diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d4dd19e..d8e428a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -17,9 +17,10 @@ use_template = process_children_only = pre_snapshot_script = post_snapshot_script = +pruning_script = +script_timeout = 5 no_inconsistent_snapshot = force_post_snapshot_script = -pruning_script = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From 6968e441468355a03bd1d6fdd39c5c93d2409d36 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 16 Oct 2018 18:10:57 +0200 Subject: [PATCH 38/51] updated documentation regarding pre/post/prun scripts --- sanoid | 1 - sanoid.conf | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 9c0e54d..69562f7 100755 --- a/sanoid +++ b/sanoid @@ -469,7 +469,6 @@ sub take_snapshots { my $dataset = (split '@', $snap)[0]; my $snapname = (split '@', $snap)[1]; my $presnapshotfailure = 0; - my $timeout = 5; if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; diff --git a/sanoid.conf b/sanoid.conf index db468e2..feb2237 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -68,19 +68,19 @@ daily_crit = 60 [template_scripts] + ### dataset and snapshot name will be supplied as environment variables + ### for all pre/post/prune scripts ($SANOID_TARGET, $SANOID_SNAPNAME) ### run script before snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET pre_snapshot_script = /path/to/script.sh ### run script after snapshot - ### dataset name will be supplied as an environment variable $SANOID_TARGET post_snapshot_script = /path/to/script.sh - ### dataset name will be supplied as an environment variable $SANOID_TARGET + ### run script after pruning snapshot pruning_script = /path/to/script.sh - ### don't take an inconsistent snapshot + ### don't take an inconsistent snapshot (skip if pre script fails) #no_inconsistent_snapshot = yes ### run post_snapshot_script when pre_snapshot_script is failing #force_post_snapshot_script = yes - ### limit allowed execution time of scripts before continuing (<= 0 -> infinite) + ### limit allowed execution time of scripts before continuing (<= 0: infinite) script_timeout = 5 [template_ignore] From 8e929a331dda6a00ab77208745b7642a2fab7d43 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 9 Nov 2018 07:31:51 +0100 Subject: [PATCH 39/51] removed sleeping between snapshot taking --- sanoid | 2 -- 1 file changed, 2 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..57acc7b 100755 --- a/sanoid +++ b/sanoid @@ -459,8 +459,6 @@ sub take_snapshots { if (!$args{'readonly'}) { system($zfs, "snapshot", "$snap") == 0 or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; - # make sure we don't end up with multiple snapshots with the same ctime - sleep 1; } } $forcecacheupdate = 1; From fa3c511dc106008bd03066059ed819f231734438 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 14 Nov 2018 18:26:11 +0100 Subject: [PATCH 40/51] split snapshot taking/pruning into seperate units for debian package to prevent pruning blocking snapshot taking --- packages/debian/rules | 12 +++++++++++- packages/debian/sanoid-prune.service | 13 +++++++++++++ packages/debian/sanoid.service | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 packages/debian/sanoid-prune.service diff --git a/packages/debian/rules b/packages/debian/rules index 83eb475..ddd77b0 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -16,4 +16,14 @@ override_dh_auto_install: @mkdir -p $(DESTDIR)/usr/share/doc/sanoid; \ cp sanoid.conf $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example; @mkdir -p $(DESTDIR)/lib/systemd/system; \ - cp debian/sanoid.timer $(DESTDIR)/lib/systemd/system; + cp debian/sanoid-prune.service $(DESTDIR)/lib/systemd/system; + +override_dh_installinit: + dh_installinit --noscripts + +override_dh_systemd_enable: + dh_systemd_enable sanoid.timer + dh_systemd_enable sanoid-prune.service + +override_dh_systemd_start: + dh_systemd_start sanoid.timer diff --git a/packages/debian/sanoid-prune.service b/packages/debian/sanoid-prune.service new file mode 100644 index 0000000..c956bd5 --- /dev/null +++ b/packages/debian/sanoid-prune.service @@ -0,0 +1,13 @@ +[Unit] +Description=Cleanup ZFS Pool +Requires=zfs.target +After=zfs.target sanoid.service +ConditionFileNotEmpty=/etc/sanoid/sanoid.conf + +[Service] +Environment=TZ=UTC +Type=oneshot +ExecStart=/usr/sbin/sanoid --prune-snapshots + +[Install] +WantedBy=sanoid.service diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service index 2d01bbf..e146354 100644 --- a/packages/debian/sanoid.service +++ b/packages/debian/sanoid.service @@ -7,4 +7,4 @@ ConditionFileNotEmpty=/etc/sanoid/sanoid.conf [Service] Environment=TZ=UTC Type=oneshot -ExecStart=/usr/sbin/sanoid --cron +ExecStart=/usr/sbin/sanoid --take-snapshots From 2796e22dbf8a6eaea8b4d1d27b04f9bea4636ddb Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 00:34:21 +0100 Subject: [PATCH 41/51] added option to defer pruning based on the available pool capacity --- sanoid | 68 +++++++++++++++++++++++++++++++++++++++++--- sanoid.defaults.conf | 4 ++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..34445aa 100755 --- a/sanoid +++ b/sanoid @@ -31,6 +31,7 @@ if (keys %args < 2) { my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; +my $zpool = '/sbin/zpool'; my $conf_file = "$args{'configdir'}/sanoid.conf"; my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; @@ -44,6 +45,7 @@ my $cache = '/var/cache/sanoidsnapshots.txt'; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); my %pruned; +my %capacitycache; my %snapsbytype = getsnapsbytype( \%config, \%snaps ); @@ -254,6 +256,10 @@ sub prune_snapshots { my $path = $config{$section}{'path'}; my $period = 0; + if (check_prune_defer($config, $section)) { + if ($args{'verbose'}) { print "INFO: deferring snapshot pruning ($section)...\n"; } + next; + } foreach my $type (keys %{ $config{$section} }){ unless ($type =~ /ly$/) { next; } @@ -872,7 +878,7 @@ sub check_zpool() { exit $ERRORS{$state}; } - my $statcommand="/sbin/zpool list -o name,size,cap,health,free $pool"; + my $statcommand="$zpool list -o name,size,cap,health,free $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -920,7 +926,7 @@ sub check_zpool() { ## flag to detect section of zpool status involving our zpool my $poolfind=0; - $statcommand="/sbin/zpool status $pool"; + $statcommand="$zpool status $pool"; if (! open STAT, "$statcommand|") { $state = 'CRITICAL'; print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n"); @@ -1028,7 +1034,7 @@ sub check_zpool() { return ($ERRORS{$state},$msg); } # end check_zpool() -sub check_capacity_limit() { +sub check_capacity_limit { my $value = shift; if (!defined($value) || $value !~ /^\d+\z/) { @@ -1051,7 +1057,7 @@ sub check_zpool_capacity() { my $capacitylimitsref=shift; my %capacitylimits=%$capacitylimitsref; - my $statcommand="/sbin/zpool list -H -o cap $pool"; + my $statcommand="$zpool list -H -o cap $pool"; if (! open STAT, "$statcommand|") { print ("$state '$statcommand' command returns no result!\n"); @@ -1096,6 +1102,60 @@ sub check_zpool_capacity() { return ($ERRORS{$state},$msg); } # end check_zpool_capacity() +sub check_prune_defer { + my ($config, $section) = @_; + + my $limit = $config{$section}{"prune_defer"}; + + if (!check_capacity_limit($limit)) { + die "ERROR: invalid prune_defer limit!\n"; + } + + if ($limit eq 0) { + return 0; + } + + my @parts = split /\//, $section, 2; + my $pool = $parts[0]; + + if (exists $capacitycache{$pool}) { + } else { + $capacitycache{$pool} = get_zpool_capacity($pool); + } + + if ($limit < $capacitycache{$pool}) { + return 0; + } + + return 1; +} + +sub get_zpool_capacity { + my $pool = shift; + + my $statcommand="$zpool list -H -o cap $pool"; + + if (! open STAT, "$statcommand|") { + die "ERROR: '$statcommand' command returns no result!\n"; + } + + my $line = ; + close(STAT); + + chomp $line; + my @row = split(/ +/, $line); + my $cap=$row[0]; + + ## check for valid capacity value + if ($cap !~ m/^[0-9]{1,3}%$/ ) { + die "ERROR: '$statcommand' command returned invalid capacity value ($cap)!\n"; + } + + $cap =~ s/\D//g; + + return $cap; +} + ###################################################################################################### ###################################################################################################### ###################################################################################################### diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0797fa8 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -26,7 +26,9 @@ hourly = 48 daily = 90 monthly = 6 yearly = 0 -min_percent_free = 10 +# pruning can be skipped based on the used capacity of the pool +# (0: always prune, 1-100: only prune if used capacity is greater than this value) +prune_defer = 0 # We will automatically take snapshots if autosnap is on, at the desired times configured # below (or immediately, if we don't have one since the last preferred time for that type). From c8b880c5e2ebb4771fa845af8e4cd006751d8970 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:01:40 +0100 Subject: [PATCH 42/51] implemented clone handling (try to recreate on target instead of full replication) --- syncoid | 116 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 23 deletions(-) diff --git a/syncoid b/syncoid index 30aef49..e9e3f27 100755 --- a/syncoid +++ b/syncoid @@ -104,17 +104,59 @@ my $exitcode = 0; ## replication ## if (!defined $args{'recursive'}) { - syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } else { if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; } my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot); - foreach my $dataset(@datasets) { + + my @deferred; + + foreach my $datasetProperties(@datasets) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + if ($origin eq "-") { + $origin = undef; + } else { + # check if clone source is replicated too + my @values = split(/@/, $origin, 2); + my $srcdataset = $values[0]; + + my $found = 0; + foreach my $datasetProperties(@datasets) { + if ($datasetProperties->{'name'} eq $srcdataset) { + $found = 1; + last; + } + } + + if ($found == 0) { + # clone source is not replicated, do a full replication + $origin = undef; + } else { + # clone source is replicated, defer until all non clones are replicated + push @deferred, $datasetProperties; + next; + } + } + $dataset =~ s/\Q$sourcefs\E//; chomp $dataset; my $childsourcefs = $sourcefs . $dataset; my $childtargetfs = $targetfs . $dataset; # print "syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); \n"; - syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); + } + + # replicate cloned datasets and if this is the initial run, recreate them on the target + foreach my $datasetProperties(@deferred) { + my $dataset = $datasetProperties->{'name'}; + my $origin = $datasetProperties->{'origin'}; + + $dataset =~ s/\Q$sourcefs\E//; + chomp $dataset; + my $childsourcefs = $sourcefs . $dataset; + my $childtargetfs = $targetfs . $dataset; + syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin); } } @@ -147,37 +189,51 @@ sub getchilddatasets { $fsescaped = escapeshellparam($fsescaped); } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name,origin -t filesystem,volume -Hr $fsescaped |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } - open FH, $getchildrencmd; - my @children = ; - close FH; - - if (defined $args{'skip-parent'}) { - # parent dataset is the first element - shift @children; + if (! open FH, $getchildrencmd) { + die "ERROR: list command failed!\n"; } - if (defined $args{'exclude'}) { - my $excludes = $args{'exclude'}; - foreach (@$excludes) { - for my $i ( 0 .. $#children ) { - if ($children[$i] =~ /$_/) { - if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; } - undef $children[$i] + my @children; + my $first = 1; + + DATASETS: while() { + chomp; + + if (defined $args{'skip-parent'} && $first eq 1) { + # parent dataset is the first element + $first = 0; + next; + } + + my ($dataset, $origin) = /^([^\t]+)\t([^\t]+)/; + + if (defined $args{'exclude'}) { + my $excludes = $args{'exclude'}; + foreach (@$excludes) { + print("$dataset\n"); + if ($dataset =~ /$_/) { + if ($debug) { print "DEBUG: excluded $dataset because of $_\n"; } + next DATASETS; } } - - @children = grep{ defined }@children; } + + my %properties; + $properties{'name'} = $dataset; + $properties{'origin'} = $origin; + + push @children, \%properties; } + close FH; return @children; } sub syncdataset { - my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_; + my ($sourcehost, $sourcefs, $targethost, $targetfs, $origin) = @_; my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); @@ -305,11 +361,25 @@ sub syncdataset { my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped"; my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + my $pvsize; + if (defined $origin) { + my $originescaped = escapeshellparam($origin); + $sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped"; + my $streamargBackup = $args{'streamarg'}; + $args{'streamarg'} = "-i"; + $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot); + $args{'streamarg'} = $streamargBackup; + } else { + $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot); + } + my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { + if (defined $origin) { + print "INFO: Clone is recreated on target $targetfs based on $origin\n"; + } if (!defined ($args{'no-stream'}) ) { print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; } else { @@ -396,7 +466,7 @@ sub syncdataset { # a resumed transfer will only be done to the next snapshot, # so do an normal sync cycle - return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef); } # find most recent matching snapshot and do an -I From 9d6cb42f4d2927c34d9d6344fbadb57f9f72caa1 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 18:08:38 +0100 Subject: [PATCH 43/51] added option to disable smart clone handling --- README.md | 5 +++++ syncoid | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b833dec..ed7a107 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to not use resumeable zfs send/receive streams. ++ --no-clone-handling + + This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + + + --dumpsnaps This prints a list of snapshots during the run. diff --git a/syncoid b/syncoid index e9e3f27..c54c915 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,8 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", + "no-clone-handling") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -114,7 +115,7 @@ if (!defined $args{'recursive'}) { foreach my $datasetProperties(@datasets) { my $dataset = $datasetProperties->{'name'}; my $origin = $datasetProperties->{'origin'}; - if ($origin eq "-") { + if ($origin eq "-" || defined $args{'no-clone-handling'}) { $origin = undef; } else { # check if clone source is replicated too @@ -1320,3 +1321,4 @@ Options: --dumpsnaps Dumps a list of snapshots during the run --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available + --no-clone-handling Don't try to recreate clones on target From f153810d08f86e804f7e43febb812ddabe810e92 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 21 Nov 2018 23:14:30 +0100 Subject: [PATCH 44/51] check for valid estimated send size to prevent a perl warning on systems which doesn't output size informations --- syncoid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncoid b/syncoid index 30aef49..1f759e2 100755 --- a/syncoid +++ b/syncoid @@ -1154,6 +1154,11 @@ sub getsendsize { } chomp $sendsize; + # check for valid value + if ($sendsize !~ /^\d+$/) { + $sendsize = ''; + } + # to avoid confusion with a zero size pv, give sendsize # a minimum 4K value - or if empty, make sure it reads UNKNOWN if ($debug) { print "DEBUG: sendsize = $sendsize\n"; } From a25ec83812ace12c26ba01296e278a2799d04d77 Mon Sep 17 00:00:00 2001 From: Rodger Donaldson Date: Sun, 25 Nov 2018 07:14:59 +1300 Subject: [PATCH 45/51] Add a dependency for the configini patch The RPM will install but fail to run if the perl-Config-IniFiles rpm is not also installed; this adds as a Requires: so that yum/dnf can find and install. --- packages/rhel/sanoid.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3a9412f..7d4995d 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -14,7 +14,7 @@ License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz -Requires: perl, mbuffer, lzop, pv +Requires: perl, mbuffer, lzop, pv, perl-Config-IniFiles %if 0%{?_with_systemd} Requires: systemd >= 212 From ea55308dfcdce7614306d8a4b7187ff61d43c4b8 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 15:51:51 +0100 Subject: [PATCH 46/51] implemented support for excluding children of a specific dataset --- sanoid | 18 ++++++++++++++---- sanoid.defaults.conf | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index 7ae1b5b..8f390ba 100755 --- a/sanoid +++ b/sanoid @@ -661,7 +661,7 @@ sub init { tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!"; # we'll use these later to normalize potentially true and false values on any toggle keys - my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only'); + my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','skip_children'); my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON"); my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF"); @@ -718,7 +718,7 @@ sub init { # override with any locally set values in the module itself foreach my $key (keys %{$ini{$section}} ) { - if (! ($key =~ /template|recursive/)) { + if (! ($key =~ /template|recursive|skip_children/)) { if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value directly set in module.\n"; } $config{$section}{$key} = $ini{$section}{$key}; } @@ -743,10 +743,17 @@ sub init { # how 'bout some recursion? =) my @datasets; - if ($ini{$section}{'recursive'}) { + if ($ini{$section}{'recursive'} || $ini{$section}{'skip_children'}) { @datasets = getchilddatasets($config{$section}{'path'}); - foreach my $dataset(@datasets) { + DATASETS: foreach my $dataset(@datasets) { chomp $dataset; + + if ($ini{$section}{'skip_children'}) { + if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; } + delete $config{$dataset}; + next DATASETS; + } + foreach my $key (keys %{$config{$section}} ) { if (! ($key =~ /template|recursive|children_only/)) { if ($args{'debug'}) { print "DEBUG: recursively setting $key from $section to $dataset.\n"; } @@ -1257,6 +1264,9 @@ sub getchilddatasets { my @children = ; close FH; + # parent dataset is the first element + shift @children; + return @children; } diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index d86cc47..0c9037a 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -15,6 +15,7 @@ path = recursive = use_template = process_children_only = +skip_children = # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately # prune any of those type snapshots already present. From a0b983ee6ef791449fb631353e26b4f05a5ab71a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 21:38:15 +0100 Subject: [PATCH 47/51] warn if unknown interval type is used --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index cb43066..5c81893 100755 --- a/sanoid +++ b/sanoid @@ -471,7 +471,7 @@ sub take_snapshots { $lastpreferred = timelocal(@preferredtime); if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year } else { - # unknown type + warn "WARN: unknown interval type $type in config!"; next; } From 7d742914e53591cfb2d13ad9dc22df55e9350446 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Wed, 23 Aug 2017 19:23:22 -0400 Subject: [PATCH 48/51] Add '--no-privilege-elevation' option to bypass root check. --- README.md | 4 ++++ syncoid | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c537540..e86d13a 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This prints a list of snapshots during the run. ++ --no-privilege-elevation + + Bypass the root check and assume syncoid has the necessary permissions (for use with ZFS permission delegation). + + --sshport Allow sync to/from boxes running SSH on non-standard ports. diff --git a/syncoid b/syncoid index e960d53..ccc5861 100755 --- a/syncoid +++ b/syncoid @@ -20,7 +20,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", - "no-clone-handling") or pod2usage(2); + "no-clone-handling", "no-privilege-elevation") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -1186,7 +1186,7 @@ sub getssh { $rhost =~ s/:\Q$fs\E$//; my $remoteuser = $rhost; $remoteuser =~ s/\@.*$//; - if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } + if ($remoteuser eq 'root' || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |"; @@ -1194,7 +1194,7 @@ sub getssh { $rhost = "-S $socket $rhost"; } else { my $localuid = $<; - if ($localuid == 0) { $isroot = 1; } else { $isroot = 0; } + if ($localuid == 0 || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; } } # if ($isroot) { print "this user is root.\n"; } else { print "this user is not root.\n"; } return ($rhost,$fs,$isroot); @@ -1455,3 +1455,4 @@ Options: --no-command-checks Do not check command existence before attempting transfer. Not recommended --no-resume Don't use the ZFS resume feature if available --no-clone-handling Don't try to recreate clones on target + --no-privilege-elevation Bypass the root check, for use with ZFS permission delegation From 23a0ce2e06a8bf091335aa8fb6ffb3356bc549a2 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Tue, 4 Dec 2018 16:33:03 -0500 Subject: [PATCH 49/51] Update CHANGELIST --- CHANGELIST | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELIST b/CHANGELIST index 515d05d..5c53199 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,29 @@ +current PRERELEASE: + #140 - --no-privilege-elevation option to bypass root checks entirely + #157 - configurable frequent snapshot period + #163 - weekly snapshot period + #247 - implement support for zfs bookmarks in syncoid + #248 - allow forced target snapshot deletion in syncoid with --force-delete, for those who like to live dangerously + #254 - add xz compression option to syncoid + #261 - add WARNings for unsupported parameters in templates + #262 - documentation typo fix + #264 - add clean WARN error for unknown snapshot type values + #265 - add support for lz4 compression to syncoid + #266 - squash uninitialized value perl warning on first sanoid run (no snapshots) + #267 - improve boolean handling for recursive parameter + #268 - update README + #269 - don't alarm for lack of hourly/daily/etc if hourly/daily/etc=0 + #277 - update automated test script for DST + #279 - FreeBSD pkg typo fix + #280 - add pre,post,prune snapshot script hooks + #286 - remove 1s sleep interval between snapshots + #287 - run snapshot prune, snapshot take as separate systemd services in Debian package + #289 - optional defer pruning until low %FREE reached + #290 - replicate clone structure from source to target + #292 - fix sendsize estimation to prevent perl warnings on some systems + #294 - add dependency for Perl::Config::Ini + #298 - exclude recursion for children of a dataset with skip-children = yes + 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! From 00a9c6f6b551ef1d19c97ecba102f9a47388fd5d Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 23:12:16 +0100 Subject: [PATCH 50/51] added changed from recently merged PR's --- CHANGELIST | 60 +++++++++++++++++++++++-------------- packages/debian/changelog | 62 ++++++++++++++++++++++++--------------- packages/rhel/sanoid.spec | 2 +- 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 3ef74ae..9172b2d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,25 +1,41 @@ -1.4.19 [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) - [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) - [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) - [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) - [syncoid] make local source bwlimit work (@phreaker0) - [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) - [sanoid] updated INSTALL with missing dependency - [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) - [sanoid] quiet flag suppresses all info output (@martinvw) - [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) - [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) - [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) - [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) - [syncoid] Added support for ZStandard compression.(@danielewood) - [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) - [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) - [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) - [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) - [sanoid] use UTC by default in unit template and documentation (@phreaker0) - [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) - [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) +1.4.19 [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) + [syncoid] added force delete flag (@phreaker0) + [sanoid] removed sleeping between snapshot taking (@phreaker0) + [syncoid] added '--no-privilege-elevation' option to bypass root check (@lopsided98) + [sanoid] implemented weekly period (@phreaker0) + [syncoid] implemented support for zfs bookmarks as fallback (@phreaker0) + [sanoid] support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0) + [sanoid] ignore snapshots types that are set to 0 (@muff1nman) + [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0) + [syncoid] replicate clones (@phreaker0) + [syncoid] added compression algorithms: lz4, xz (@spheenik, @phreaker0) + [sanoid] added option to defer pruning based on the available pool capacity (@phreaker0) + [sanoid] implemented frequent snapshots with configurable period (@phreaker0) + [syncoid] prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0) + [packaging] dependency fixes (@rodgerd, mabushey) + [syncoid] implemented support for excluding children of a specific dataset (@phreaker0) + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid, thank you @phreaker0! diff --git a/packages/debian/changelog b/packages/debian/changelog index 67cca4d..34909b6 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,29 +1,45 @@ sanoid (1.4.19) unstable; urgency=medium - [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) - [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) - [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) - [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) - [syncoid] make local source bwlimit work (@phreaker0) - [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) - [sanoid] updated INSTALL with missing dependency - [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) - [sanoid] quiet flag suppresses all info output (@martinvw) - [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) - [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) - [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) - [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) - [syncoid] Added support for ZStandard compression.(@danielewood) - [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) - [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) - [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) - [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) - [sanoid] use UTC by default in unit template and documentation (@phreaker0) - [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) - [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) + [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) + [syncoid] added force delete flag (@phreaker0) + [sanoid] removed sleeping between snapshot taking (@phreaker0) + [syncoid] added '--no-privilege-elevation' option to bypass root check (@lopsided98) + [sanoid] implemented weekly period (@phreaker0) + [syncoid] implemented support for zfs bookmarks as fallback (@phreaker0) + [sanoid] support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0) + [sanoid] ignore snapshots types that are set to 0 (@muff1nman) + [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0) + [syncoid] replicate clones (@phreaker0) + [syncoid] added compression algorithms: lz4, xz (@spheenik, @phreaker0) + [sanoid] added option to defer pruning based on the available pool capacity (@phreaker0) + [sanoid] implemented frequent snapshots with configurable period (@phreaker0) + [syncoid] prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0) + [packaging] dependency fixes (@rodgerd, mabushey) + [syncoid] implemented support for excluding children of a specific dataset (@phreaker0) + [sanoid] monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0) + [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie) + [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0) + [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0) + [syncoid] make local source bwlimit work (@phreaker0) + [syncoid] fix 'resume support' detection on FreeBSD (@pit3k) + [sanoid] updated INSTALL with missing dependency + [sanoid] fixed monitor-health command for pools containing cache and log devices (@phreaker0) + [sanoid] quiet flag suppresses all info output (@martinvw) + [sanoid] check for empty lockfile which lead to sanoid failing on start (@jasonblewis) + [sanoid] added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0) + [sanoid] cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0) + [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) + [syncoid] Added support for ZStandard compression.(@danielewood) + [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) + [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) + [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) + [sanoid] use UTC by default in unit template and documentation (@phreaker0) + [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0) + [syncoid] documented compatibility issues with (t)csh shells (@ecoutu) - -- Jim Salter Wed, 05 Sep 2018 04:00:00 -0400 + -- Jim Salter Wed, 04 Dec 2018 18:10:00 -0400 sanoid (1.4.18) unstable; urgency=medium diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index b5d7172..1579a27 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -111,7 +111,7 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog -* Wed Sep 05 2018 Christoph Klaffl - 1.4.19 +* Wed Dec 04 2018 Christoph Klaffl - 1.4.19 - Bump to 1.4.19 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 - Bump to 1.4.18 From 3a1ffe8554802dfc2459032b3beba5f5cf360640 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 4 Dec 2018 23:33:18 +0100 Subject: [PATCH 51/51] fixed a regression which causes perl warnings --- sanoid | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index fbb94b6..dc71b9d 100755 --- a/sanoid +++ b/sanoid @@ -843,13 +843,15 @@ sub init { } # how 'bout some recursion? =) + my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue ); + my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue ); my @datasets; - if (grep( /^$ini{$section}{'recursive'}$/, @istrue ) || grep( /^$ini{$section}{'skip_children'}$/, @istrue )) { + if ($recursive || $skipChildren) { @datasets = getchilddatasets($config{$section}{'path'}); DATASETS: foreach my $dataset(@datasets) { chomp $dataset; - if (grep( /^$ini{$section}{'skip_children'}$/, @istrue )) { + if ($skipChildren) { if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; } delete $config{$dataset}; next DATASETS;