From d8613d13797db640f9cf7432f29677056bddeb43 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 7 Nov 2017 08:03:53 +0100
Subject: [PATCH 01/68] implemented frequent snapshots with configurable period

Fixes #75
---
 sanoid               | 21 +++++++++++++++++----
 sanoid.conf          |  2 ++
 sanoid.defaults.conf |  7 +++++++
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/sanoid b/sanoid
index d6e58ce..e8756db 100755
--- a/sanoid
+++ b/sanoid
@@ -121,12 +121,13 @@ sub monitor_snapshots {
 		my $path = $config{$section}{'path'};
 		push @paths, $path;
 
-		my @types = ('yearly','monthly','daily','hourly');
+		my @types = ('yearly','monthly','daily','hourly','frequently');
 		foreach my $type (@types) {
 
 			my $smallerperiod = 0;
 			# we need to set the period length in seconds first
-			if ($type eq 'hourly') { $smallerperiod = 60; }
+			if ($type eq 'frequently') { $smallerperiod = 1; }
+			elsif ($type eq 'hourly') { $smallerperiod = 60; }
 			elsif ($type eq 'daily') { $smallerperiod = 60*60; }
 			elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; }
 			elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; }
@@ -200,7 +201,8 @@ sub prune_snapshots {
 			unless ($type =~ /ly$/) { next; }
 
 			# we need to set the period length in seconds first
-			if ($type eq 'hourly') { $period = 60*60; }
+			if ($type eq 'frequently') { $period = 60 * $config{$section}{'frequent_period'}; }
+			elsif ($type eq 'hourly') { $period = 60*60; }
 			elsif ($type eq 'daily') { $period = 60*60*24; }
 			elsif ($type eq 'monthly') { $period = 60*60*24*31; }
 			elsif ($type eq 'yearly') { $period = 60*60*24*365.25; }
@@ -291,7 +293,18 @@ sub take_snapshots {
 				my @preferredtime;
 				my $lastpreferred;
 
-				if ($type eq 'hourly')     {
+				if ($type eq 'frequently')     {
+					my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'});
+
+					push @preferredtime,0; # try to hit 0 seconds
+					push @preferredtime,$frequentslice * $config{$section}{'frequent_period'};
+					push @preferredtime,$datestamp{'hour'};
+					push @preferredtime,$datestamp{'mday'};
+					push @preferredtime,($datestamp{'mon'}-1); # january is month 0
+					push @preferredtime,$datestamp{'year'};
+					$lastpreferred = timelocal(@preferredtime);
+					if ($lastpreferred > time()) { $lastpreferred -= 60 * $config{$section}{'frequent_period'}; } # preferred time is later this frequent period - so look at last frequent period
+				} elsif ($type eq 'hourly')     {
 					push @preferredtime,0; # try to hit 0 seconds
 					push @preferredtime,$config{$section}{'hourly_min'};
 					push @preferredtime,$datestamp{'hour'};
diff --git a/sanoid.conf b/sanoid.conf
index 9b1f19d..b999634 100644
--- a/sanoid.conf
+++ b/sanoid.conf
@@ -40,6 +40,7 @@
 	daily = 60
 
 [template_production]
+	frequently = 0
 	hourly = 36
 	daily = 30
 	monthly = 3
@@ -49,6 +50,7 @@
 
 [template_backup]
 	autoprune = yes
+	frequently = 0
 	hourly = 30
 	daily = 90
 	monthly = 12
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 35c804d..b5e4e63 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -16,12 +16,17 @@ recursive =
 use_template =
 process_children_only =
 
+# The period in minutes for frequent snapshots,
+# should be in the range of 1-30 and divide an hour without remainder
+frequent_period = 15
+
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
 # prune any of those type snapshots already present.
 #
 # Otherwise, if autoprune is set, we will prune any snapshots of that type which are older
 # than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days.
 autoprune = yes
+frequently = 0
 hourly = 48
 daily = 90
 monthly = 6
@@ -62,6 +67,8 @@ yearly_min = 0
 monitor = yes
 monitor_dont_warn = no
 monitor_dont_crit = no
+frequently_warn = 2000
+frequently_crit = 8000
 hourly_warn = 90
 hourly_crit = 360
 daily_warn = 28

From c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 8 Nov 2017 20:25:07 +0100
Subject: [PATCH 02/68] hardcoded new defaults

---
 sanoid | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/sanoid b/sanoid
index e8756db..6428cdf 100755
--- a/sanoid
+++ b/sanoid
@@ -553,6 +553,18 @@ sub getsnaps {
 ####################################################################################
 ####################################################################################
 
+sub verify_option_existence {
+    my ($hash, $key, $default) = @_;
+
+    if (! defined (%$hash{$key})) {
+       $hash->{$key} = $default;
+    }
+}
+
+####################################################################################
+####################################################################################
+####################################################################################
+
 sub init {
 	my ($conf_file, $default_conf_file) = @_;
 	my %config;
@@ -568,6 +580,12 @@ sub init {
 	my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
 	my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
 
+	# hardcoded defaults which may be missing from older default configuration file
+	verify_option_existence($defaults{'template_default'}, 'frequent_period', 15);
+	verify_option_existence($defaults{'template_default'}, 'frequently', 0);
+	verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000);
+	verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000);
+
 	foreach my $section (keys %ini) {
 
 		# first up - die with honor if unknown parameters are set in any modules or templates by the user.

From 8bd98f18008494e27c47c6d16cc1fbbaec9a33a6 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 8 Nov 2017 21:40:30 +0100
Subject: [PATCH 03/68] added more documentation for frequent snapshots

---
 README.md            |  1 +
 sanoid.defaults.conf | 12 ++++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9c4e6ba..863b697 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,7 @@ And its /etc/sanoid/sanoid.conf might look something like this:
 #############################
 
 [template_production]
+        frequently = 0
         hourly = 36
         daily = 30
         monthly = 3
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index b5e4e63..e7c22a8 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -16,8 +16,16 @@ recursive =
 use_template =
 process_children_only =
 
-# The period in minutes for frequent snapshots,
-# should be in the range of 1-30 and divide an hour without remainder
+# for snapshots shorter than one hour, the period duration must be defined
+# in minutes. Because they are executed within a full hour, the selected
+# value should divide 60 minutes without remainder so taken snapshots
+# are apart in equal intervals. Values larger than 59 aren't practical
+# as only one snapshot will be taken on each full hour in this case.
+# examples:
+# frequent_period = 15 -> four snapshot each hour 15 minutes apart
+# frequent_period = 5 -> twelve snapshots each hour 5 minutes apart
+# frequent_period = 45 -> two snapshots each hour with different time gaps
+# between them: 45 minutes and 15 minutes in this case
 frequent_period = 15
 
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately

From bc2f8bd4e9b150294bad5c3150a682a3e857e3e0 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 8 Nov 2017 21:48:37 +0100
Subject: [PATCH 04/68] Revert "hardcoded new defaults"

This reverts commit c9adcdab1e7e2a7eaa04bdbc15a24881278cefdb.
---
 sanoid | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/sanoid b/sanoid
index 6428cdf..e8756db 100755
--- a/sanoid
+++ b/sanoid
@@ -553,18 +553,6 @@ sub getsnaps {
 ####################################################################################
 ####################################################################################
 
-sub verify_option_existence {
-    my ($hash, $key, $default) = @_;
-
-    if (! defined (%$hash{$key})) {
-       $hash->{$key} = $default;
-    }
-}
-
-####################################################################################
-####################################################################################
-####################################################################################
-
 sub init {
 	my ($conf_file, $default_conf_file) = @_;
 	my %config;
@@ -580,12 +568,6 @@ sub init {
 	my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
 	my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
 
-	# hardcoded defaults which may be missing from older default configuration file
-	verify_option_existence($defaults{'template_default'}, 'frequent_period', 15);
-	verify_option_existence($defaults{'template_default'}, 'frequently', 0);
-	verify_option_existence($defaults{'template_default'}, 'frequently_warn', 2000);
-	verify_option_existence($defaults{'template_default'}, 'frequently_crit', 8000);
-
 	foreach my $section (keys %ini) {
 
 		# first up - die with honor if unknown parameters are set in any modules or templates by the user.

From 293d83bdaa6155e765432a9a145a786d72afed9e Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 8 Nov 2017 22:08:02 +0100
Subject: [PATCH 05/68] versioning and compatibility check for default
 configuration file

---
 sanoid               | 12 ++++++++++++
 sanoid.defaults.conf |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/sanoid b/sanoid
index e8756db..452f535 100755
--- a/sanoid
+++ b/sanoid
@@ -5,6 +5,7 @@
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
 $::VERSION = '1.4.17';
+my $MINIMUM_DEFAULTS_VERSION = 2;
 
 use strict;
 use warnings;
@@ -568,6 +569,17 @@ sub init {
 	my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
 	my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
 
+	# check if default configuration file is up to date
+	my $defaults_version = 1;
+	if (defined $defaults{'version'}{'version'}) {
+		$defaults_version = $defaults{'version'}{'version'};
+		delete $defaults{'version'};
+	}
+
+	if ($defaults_version < $MINIMUM_DEFAULTS_VERSION) {
+		die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION";
+	}
+
 	foreach my $section (keys %ini) {
 
 		# first up - die with honor if unknown parameters are set in any modules or templates by the user.
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index e7c22a8..06fc714 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -5,6 +5,8 @@
 #                                                                                 #
 #                    you have been warned.                                        #
 ###################################################################################
+[version]
+version = 2
 
 [template_default]
 

From 00b920682913641732981c233acd9262671719af Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Thu, 9 Nov 2017 09:44:40 +0100
Subject: [PATCH 06/68] implemented weekly period

---
 sanoid               | 26 +++++++++++++++++++++++---
 sanoid.defaults.conf | 11 +++++++++--
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/sanoid b/sanoid
index d6e58ce..9b6c815 100755
--- a/sanoid
+++ b/sanoid
@@ -121,15 +121,16 @@ sub monitor_snapshots {
 		my $path = $config{$section}{'path'};
 		push @paths, $path;
 
-		my @types = ('yearly','monthly','daily','hourly');
+		my @types = ('yearly','monthly', 'weekly', 'daily','hourly');
 		foreach my $type (@types) {
 
 			my $smallerperiod = 0;
 			# we need to set the period length in seconds first
 			if ($type eq 'hourly') { $smallerperiod = 60; }
 			elsif ($type eq 'daily') { $smallerperiod = 60*60; }
-			elsif ($type eq 'monthly') { $smallerperiod = 60*60*24; }
-			elsif ($type eq 'yearly') { $smallerperiod = 60*60*24; }
+			elsif ($type eq 'weekly') { $smallerperiod = 60*60*24; }
+			elsif ($type eq 'monthly') { $smallerperiod = 60*60*24*7; }
+			elsif ($type eq 'yearly') { $smallerperiod = 60*60*24*31; }
 
 			my $typewarn = $type . '_warn';
 			my $typecrit = $type . '_crit';
@@ -202,6 +203,7 @@ sub prune_snapshots {
 			# we need to set the period length in seconds first
 			if ($type eq 'hourly') { $period = 60*60; }
 			elsif ($type eq 'daily') { $period = 60*60*24; }
+			elsif ($type eq 'weekly') { $period = 60*60*24*7; }
 			elsif ($type eq 'monthly') { $period = 60*60*24*31; }
 			elsif ($type eq 'yearly') { $period = 60*60*24*365.25; }
 
@@ -309,6 +311,24 @@ sub take_snapshots {
 					push @preferredtime,$datestamp{'year'};
 					$lastpreferred = timelocal(@preferredtime);
 					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's
+				} elsif ($type eq 'weekly')   {
+					# calculate offset in seconds for the desired weekday
+					my $offset;
+					if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) {
+						$offset += 6;
+					}
+					$offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'};
+					$offset *= 60*60*24; # full day
+
+					push @preferredtime,0; # try to hit 0 seconds
+					push @preferredtime,$config{$section}{'weekly_min'};
+					push @preferredtime,$config{$section}{'weekly_hour'};
+					push @preferredtime,$datestamp{'mday'};
+					push @preferredtime,($datestamp{'mon'}-1); # january is month 0
+					push @preferredtime,$datestamp{'year'};
+					$lastpreferred = timelocal(@preferredtime);
+					$lastpreferred -= $offset;
+					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's
 				} elsif ($type eq 'monthly') {
 					push @preferredtime,0; # try to hit 0 seconds
 					push @preferredtime,$config{$section}{'monthly_min'};
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 35c804d..7187104 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -24,6 +24,7 @@ process_children_only =
 autoprune = yes
 hourly = 48
 daily = 90
+weekly = 0
 monthly = 6
 yearly = 0
 min_percent_free = 10
@@ -40,6 +41,10 @@ hourly_min = 0
 # daily - at 23:59 (most people expect a daily to contain everything done DURING that day)
 daily_hour = 23
 daily_min = 59
+# weekly -at 23:30 each Monday
+weekly_wday = 1
+weekly_hour = 23
+weekly_min = 30
 # monthly - immediately at the beginning of the month (ie 00:00 of day 1)
 monthly_mday = 1
 monthly_hour = 0
@@ -66,7 +71,9 @@ hourly_warn = 90
 hourly_crit = 360
 daily_warn = 28
 daily_crit = 32
-monthly_warn = 32
-monthly_crit = 35
+weekly_warn = 7
+weekly_crit = 10
+monthly_warn = 5
+monthly_crit = 6
 yearly_warn = 0
 yearly_crit = 0

From 4649704046cee190c8482e00e79b9b8d408829e4 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Thu, 9 Nov 2017 17:41:52 +0100
Subject: [PATCH 07/68] codestyle fix and disable monitoring of weekly
 snapshots

---
 sanoid               | 2 +-
 sanoid.defaults.conf | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sanoid b/sanoid
index 9b6c815..05c4def 100755
--- a/sanoid
+++ b/sanoid
@@ -121,7 +121,7 @@ sub monitor_snapshots {
 		my $path = $config{$section}{'path'};
 		push @paths, $path;
 
-		my @types = ('yearly','monthly', 'weekly', 'daily','hourly');
+		my @types = ('yearly','monthly','weekly','daily','hourly');
 		foreach my $type (@types) {
 
 			my $smallerperiod = 0;
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 7187104..ff79ebb 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -71,8 +71,8 @@ hourly_warn = 90
 hourly_crit = 360
 daily_warn = 28
 daily_crit = 32
-weekly_warn = 7
-weekly_crit = 10
+weekly_warn = 0
+weekly_crit = 0
 monthly_warn = 5
 monthly_crit = 6
 yearly_warn = 0

From ac16b2128e6863a8af619d0bf9282bf92f6dfbc9 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Thu, 9 Nov 2017 17:44:04 +0100
Subject: [PATCH 08/68] disable monitoring of frequent snapshots

---
 sanoid.defaults.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 06fc714..a521401 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -77,8 +77,8 @@ yearly_min = 0
 monitor = yes
 monitor_dont_warn = no
 monitor_dont_crit = no
-frequently_warn = 2000
-frequently_crit = 8000
+frequently_warn = 0
+frequently_crit = 0
 hourly_warn = 90
 hourly_crit = 360
 daily_warn = 28

From 2a3d91e4465ea35ce68548915aeaf4eb1ec75745 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 13 Dec 2017 00:44:48 +0100
Subject: [PATCH 09/68] fixed weekly snapshot interval

---
 sanoid | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sanoid b/sanoid
index 05c4def..792497a 100755
--- a/sanoid
+++ b/sanoid
@@ -313,9 +313,9 @@ sub take_snapshots {
 					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24; } # preferred time is later today - so look at yesterday's
 				} elsif ($type eq 'weekly')   {
 					# calculate offset in seconds for the desired weekday
-					my $offset;
+					my $offset = 0;
 					if ($config{$section}{'weekly_wday'} < $datestamp{'wday'}) {
-						$offset += 6;
+						$offset += 7;
 					}
 					$offset += $config{$section}{'weekly_wday'} - $datestamp{'wday'};
 					$offset *= 60*60*24; # full day
@@ -327,7 +327,7 @@ sub take_snapshots {
 					push @preferredtime,($datestamp{'mon'}-1); # january is month 0
 					push @preferredtime,$datestamp{'year'};
 					$lastpreferred = timelocal(@preferredtime);
-					$lastpreferred -= $offset;
+					$lastpreferred += $offset;
 					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*7; } # preferred time is later today - so look at yesterday's
 				} elsif ($type eq 'monthly') {
 					push @preferredtime,0; # try to hit 0 seconds

From 3d9b3b04badbce3074c279eb34aab97977df47b1 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Thu, 4 Jan 2018 12:20:37 +0100
Subject: [PATCH 10/68] Added "force-prune" option to skip busy dataset check

---
 sanoid | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sanoid b/sanoid
index d6e58ce..029d846 100755
--- a/sanoid
+++ b/sanoid
@@ -18,7 +18,7 @@ use Time::Local;      # to parse dates in reverse
 my %args = ("configdir" => "/etc/sanoid");
 GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet",
                    "monitor-health", "force-update", "configdir=s",
-                   "monitor-snapshots", "take-snapshots", "prune-snapshots"
+                   "monitor-snapshots", "take-snapshots", "prune-snapshots", "force-prune"
           ) or pod2usage(2);
 
 # If only config directory (or nothing) has been specified, default to --cron --verbose
@@ -234,7 +234,7 @@ sub prune_snapshots {
 						writelock('sanoid_pruning');
 						foreach my $snap( @prunesnaps ){
 							if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
-							if (iszfsbusy($path)) {
+							if (iszfsbusy($path) && !$args{'force-prune'}) {
 								print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n";
 							} else {
 								if (! $args{'readonly'}) { system($zfs, "destroy",$snap) == 0 or warn "could not remove $snap : $?"; }
@@ -1082,6 +1082,7 @@ Options:
   --monitor-snapshots   Reports on snapshot "health", in a Nagios compatible format
   --take-snapshots      Creates snapshots as specified in sanoid.conf
   --prune-snapshots     Purges expired snapshots as specified in sanoid.conf
+  --force-prune		Purges expired snapshots even if a send/recv is in progress
 
   --help                Prints this helptext
   --version             Prints the version number

From f670631ca281961c318b0b9fb56ca28017e5de38 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 30 Apr 2018 21:50:17 +0200
Subject: [PATCH 11/68] Added "no-clone-rollback" option to prevent clone
 rollback on target host

---
 syncoid | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/syncoid b/syncoid
index 7337f5b..e91c126 100755
--- a/syncoid
+++ b/syncoid
@@ -19,7 +19,7 @@ use Sys::Hostname;
 my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
-                   "debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2);
+                   "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 
@@ -306,13 +306,17 @@ sub syncdataset {
 			if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; }
 		} else {
 			# rollback target to matchingsnap
+			my $rollbacktype="-R";
+			if (defined $args{'no-clone-rollback'}) {
+				$rollbacktype = "-r";
+			}
 			if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
 			if ($targethost ne '') {
-				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; }
-				system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap");
+				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
+				system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap");
 			} else {
-				if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; }
-				system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap");
+				if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
+				system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap");
 			}
 
 			my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap";
@@ -949,6 +953,7 @@ Options:
   --target-bwlimit=<limit k|m|g|t>  Bandwidth limit on the target transfer
   --no-stream           Replicates using newest snapshot instead of intermediates
   --no-sync-snap        Does not create new snapshot, only transfers existing
+  --no-clone-rollback	Does not rollback clones on target
 
   --sshkey=FILE         Specifies a ssh public key to use to connect
   --sshport=PORT        Connects to remote on a particular port

From 670e76458c0dff00323aa4ceff67b16cb2fc1841 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 30 Apr 2018 22:45:05 +0200
Subject: [PATCH 12/68] Reintroduce "no-resume", erroneously removed from
 previous commit

---
 syncoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncoid b/syncoid
index 512e153..c6c85ed 100755
--- a/syncoid
+++ b/syncoid
@@ -19,7 +19,7 @@ use Sys::Hostname;
 my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
-                   "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback") or pod2usage(2);
+                   "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-resume") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 

From 793f1c782b703a9050f95a06eb8d6847af58f64e Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Tue, 1 May 2018 09:32:18 +0200
Subject: [PATCH 13/68] Reintroduced escapeshellparam for remote command and
 updated README file

---
 README.md | 8 ++++++++
 syncoid   | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cc75bdf..7a9b76c 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da
 
 	This will process your sanoid.conf file, it will NOT create snapshots, but it will purge expired ones.
 
++ --force-prune
+
+	Purges expired snapshots even if a send/recv is in progress
+
 + --monitor-snapshots
 
 	This option is designed to be run by a Nagios monitoring system. It reports on the health of your snapshots.
@@ -150,6 +154,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to restrict itself to existing snapshots, instead of creating a semi-ephemeral syncoid snapshot at execution time. Especially useful in multi-target (A->B, A->C) replication schemes, where you might otherwise accumulate a large number of foreign syncoid snapshots.
 
++ --no-clone-rollback
+
+	Does not rollback clones on target
+
 + --no-resume
 
 	This argument tells syncoid to not use resumeable zfs send/receive streams.
diff --git a/syncoid b/syncoid
index c6c85ed..0a2fe4a 100755
--- a/syncoid
+++ b/syncoid
@@ -366,7 +366,7 @@ sub syncdataset {
 			if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
 			if ($targethost ne '') {
 				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
-				system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap");
+				system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"));
 			} else {
 				if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
 				system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap");

From 679f1f3bda6feec0392eada8ae65072ac0393627 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Tue, 1 May 2018 09:39:35 +0200
Subject: [PATCH 14/68] Reintroduced targetfsescaped

---
 syncoid | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/syncoid b/syncoid
index 0a2fe4a..49e3aac 100755
--- a/syncoid
+++ b/syncoid
@@ -359,19 +359,18 @@ sub syncdataset {
 		} else {
 			my $matchingsnapescaped = escapeshellparam($matchingsnap);
 			# rollback target to matchingsnap
-			my $rollbacktype="-R";
+			my $rollbacktype = "-R";
 			if (defined $args{'no-clone-rollback'}) {
 				$rollbacktype = "-r";
 			}
 			if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
 			if ($targethost ne '') {
-				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
+				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
 				system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"));
 			} else {
-				if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap\n"; }
-				system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfs\@$matchingsnap");
+				if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
+				system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped");
 			}
-
 			my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
 			my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
 			my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);

From f9049085c8b9d7ff4800c44d4cd286bd67337107 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 11 Jun 2018 09:48:45 +0200
Subject: [PATCH 15/68] Added "no-rollback" option to prevent any rollback on
 target host

---
 syncoid | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/syncoid b/syncoid
index 49e3aac..9dc83fa 100755
--- a/syncoid
+++ b/syncoid
@@ -19,7 +19,7 @@ use Sys::Hostname;
 my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
-                   "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-resume") or pod2usage(2);
+                   "debug", "quiet", "no-stream", "no-sync-snap", "no-clone-rollback", "no-rollback", "no-resume") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 
@@ -151,6 +151,12 @@ sub syncdataset {
 	my $sourcefsescaped = escapeshellparam($sourcefs);
 	my $targetfsescaped = escapeshellparam($targetfs);
 
+	# if no rollbacks are allowed, disable forced receive
+	my $forcedrecv = "-F";
+	if (defined $args{'no-rollback'}) {
+		$forcedrecv = "";
+	}
+
 	if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; }
 
 	# make sure target is not currently in receive.
@@ -244,7 +250,7 @@ sub syncdataset {
 		my $oldestsnapescaped = escapeshellparam($oldestsnap);
 
 		my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped";
-		my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+		my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 
 		my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot);
 		my $disp_pvsize = readablebytes($pvsize);
@@ -313,7 +319,7 @@ sub syncdataset {
 		# snapshot, do a normal sync after that
 		if (defined($receivetoken)) {
 		    my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken";
-		    my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+		    my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 		    my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken);
 		    my $disp_pvsize = readablebytes($pvsize);
 		    if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
@@ -359,20 +365,22 @@ sub syncdataset {
 		} else {
 			my $matchingsnapescaped = escapeshellparam($matchingsnap);
 			# rollback target to matchingsnap
-			my $rollbacktype = "-R";
-			if (defined $args{'no-clone-rollback'}) {
-				$rollbacktype = "-r";
-			}
-			if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
-			if ($targethost ne '') {
-				if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
-				system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"));
-			} else {
-				if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
-				system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped");
+			if (!defined $args{'no-rollback'}) {
+				my $rollbacktype = "-R";
+				if (defined $args{'no-clone-rollback'}) {
+					$rollbacktype = "-r";
+				}
+				if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; }
+				if ($targethost ne '') {
+					if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
+					system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped"));
+				} else {
+					if ($debug) { print "$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped\n"; }
+					system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped");
+				}
 			}
 			my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
-			my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+			my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 			my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
 			my $disp_pvsize = readablebytes($pvsize);
 			if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
@@ -1143,6 +1151,7 @@ Options:
   --no-stream           Replicates using newest snapshot instead of intermediates
   --no-sync-snap        Does not create new snapshot, only transfers existing
   --no-clone-rollback	Does not rollback clones on target
+  --no-rollback         Does not rollback clones or snapshots on target (it probably requires a readonly target)
 
   --sshkey=FILE         Specifies a ssh public key to use to connect
   --sshport=PORT        Connects to remote on a particular port

From 4ed6ff0e447f436fb3b7a77405050839927e61a6 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 11 Jun 2018 10:06:36 +0200
Subject: [PATCH 16/68] Updated README file

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7a9b76c..58ec7dd 100644
--- a/README.md
+++ b/README.md
@@ -156,7 +156,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 + --no-clone-rollback
 
-	Does not rollback clones on target
+	Do not rollback clones on target
+
++ --no-rollback
+	Do not rollback anything (clones or snapshots) on target host
 
 + --no-resume
 

From 0aaac4205790e1e179d41140ba38142cf442389c Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 11 Jun 2018 10:16:18 +0200
Subject: [PATCH 17/68] skip pruning with --no-sync-snap (as by PR #218)

---
 syncoid | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/syncoid b/syncoid
index 9dc83fa..ae31131 100755
--- a/syncoid
+++ b/syncoid
@@ -398,9 +398,11 @@ sub syncdataset {
 		}
 	}
 
-	# prune obsolete sync snaps on source and target.
-	pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}});
-	pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}});
+	if (!defined $args{'no-sync-snap'}) {
+		# prune obsolete sync snaps on source and target (only if this run created ones).
+		pruneoldsyncsnaps($sourcehost,$sourcefs,$newsyncsnap,$sourceisroot,keys %{ $snaps{'source'}});
+		pruneoldsyncsnaps($targethost,$targetfs,$newsyncsnap,$targetisroot,keys %{ $snaps{'target'}});
+	}
 
 } # end syncdataset()
 

From add7bf6de769e32a35f200c87143f4dfcbc126dc Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Tue, 19 Jun 2018 19:08:30 +0200
Subject: [PATCH 18/68] Added a missing newline in README

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 58ec7dd..4215ee4 100644
--- a/README.md
+++ b/README.md
@@ -159,6 +159,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 	Do not rollback clones on target
 
 + --no-rollback
+
 	Do not rollback anything (clones or snapshots) on target host
 
 + --no-resume

From 99dcf7f3406b64f7f6ae261b41666fd2bbbaeec9 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 27 Jul 2018 22:52:36 +0200
Subject: [PATCH 19/68] implemented support for zfs bookmarks, if no matching
 snapshots are found bookmarks are tried as fallback, both stream and
 no-stream cases are supported

---
 syncoid | 180 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 148 insertions(+), 32 deletions(-)

diff --git a/syncoid b/syncoid
index 6c569b2..3cbff6b 100755
--- a/syncoid
+++ b/syncoid
@@ -370,11 +370,48 @@ sub syncdataset {
 
 		my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
 
-		my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps);
+		my $bookmark = 0;
+		my $bookmarkcreation = 0;
+
+		my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
 		if (! $matchingsnap) {
-			# no matching snapshot; we whined piteously already, but let's go ahead and return false
-			# now in case more child datasets need replication.
-			return 0;
+			# no matching snapshots, check for bookmarks as fallback
+			my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
+
+			# check for matching guid of source bookmark and target snapshot (oldest first)
+			foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) {
+				my $guid = $snaps{'target'}{$snap}{'guid'};
+
+				if (defined $bookmarks{$guid}) {
+					# found a match
+					$bookmark = $bookmarks{$guid}{'name'};
+					$bookmarkcreation = $bookmarks{$guid}{'creation'};
+					$matchingsnap = $snap;
+					last;
+				}
+			}
+
+			if (! $bookmark) {
+				# if we got this far, we failed to find a matching snapshot/bookmark.
+
+				print "\n";
+				print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n";
+				print "                Replication to target would require destroying existing\n";
+				print "                target. Cowardly refusing to destroy your existing target.\n\n";
+
+				# experience tells me we need a mollyguard for people who try to
+				# zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ...
+
+				if ( $targetsize < (64*1024*1024) ) {
+					print "          NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n";
+					print "                \`zfs create $args{'target'}\` on the target? ZFS initial\n";
+					print "                replication must be to a NON EXISTENT DATASET, which will\n";
+					print "                then be CREATED BY the initial replication process.\n\n";
+				}
+
+				# return false now in case more child datasets need replication.
+				return 0;
+			}
 		}
 
 		# make sure target is (still) not currently in receive.
@@ -398,17 +435,68 @@ sub syncdataset {
 				system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped");
 			}
 
-			my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
-			my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
-			my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
-			my $disp_pvsize = readablebytes($pvsize);
-			if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
-			my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
+			my $nextsnapshot = 0;
 
-			if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
-			if ($debug) { print "DEBUG: $synccmd\n"; }
-			system("$synccmd") == 0
-				or die "CRITICAL ERROR: $synccmd failed: $?";
+			if ($bookmark) {
+				my $bookmarkescaped = escapeshellparam($bookmark);
+
+				if (!defined $args{'no-stream'}) {
+					# if intermediate snapshots are needed we need to find the next oldest snapshot,
+					# do an replication to it and replicate as always from oldest to newest
+					# because bookmark sends doesn't support intermediates directly
+					foreach my $snap ( sort { $snaps{'source'}{$a}{'creation'}<=>$snaps{'source'}{$b}{'creation'} } keys %{ $snaps{'source'} }) {
+						if ($snaps{'source'}{$snap}{'creation'} >= $bookmarkcreation) {
+							$nextsnapshot = $snap;
+							last;
+						}
+					}
+				}
+
+				# bookmark stream size can't be determined
+				my $pvsize = 0;
+				my $disp_pvsize = "UNKNOWN";
+
+				if ($nextsnapshot) {
+					my $nextsnapshotescaped = escapeshellparam($nextsnapshot);
+
+					my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped";
+					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+					my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
+
+					if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; }
+					if ($debug) { print "DEBUG: $synccmd\n"; }
+					system("$synccmd") == 0
+						or die "CRITICAL ERROR: $synccmd failed: $?";
+
+					$matchingsnap = $nextsnapshot;
+					$matchingsnapescaped = escapeshellparam($matchingsnap);
+				} else {
+					my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped";
+					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+					my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
+
+					if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; }
+					if ($debug) { print "DEBUG: $synccmd\n"; }
+					system("$synccmd") == 0
+						or die "CRITICAL ERROR: $synccmd failed: $?";
+				}
+			}
+
+			# do a normal replication if bookmarks aren't used or if previous
+			# bookmark replication was only done to the next oldest snapshot
+			if (!$bookmark || $nextsnapshot) {
+				my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
+				my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+				my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
+				my $disp_pvsize = readablebytes($pvsize);
+				if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
+				my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
+
+				if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
+				if ($debug) { print "DEBUG: $synccmd\n"; }
+				system("$synccmd") == 0
+					or die "CRITICAL ERROR: $synccmd failed: $?";
+			}
 
 			# restore original readonly value to target after sync complete
 				# dyking this functionality out for the time being due to buggy mount/unmount behavior
@@ -900,31 +988,15 @@ sub pruneoldsyncsnaps {
 }
 
 sub getmatchingsnapshot {
-	my ($sourcefs, $targetfs, $targetsize, $snaps) = @_;
+	my ($sourcefs, $targetfs, $snaps) = @_;
 	foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) {
-		if (defined $snaps{'target'}{$snap}{'guid'}) {
+		if (defined $snaps{'target'}{$snap}) {
 			if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) {
 				return $snap;
 			}
 		}
 	}
 
-	# if we got this far, we failed to find a matching snapshot.
-
-	print "\n";
-	print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n";
-	print "                Replication to target would require destroying existing\n";
-	print "                target. Cowardly refusing to destroy your existing target.\n\n";
-
-	# experience tells me we need a mollyguard for people who try to
-	# zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ...
-
-	if ( $targetsize < (64*1024*1024) ) {
-		print "          NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n";
-		print "                \`zfs create $args{'target'}\` on the target? ZFS initial\n";
-		print "                replication must be to a NON EXISTENT DATASET, which will\n";
-		print "                then be CREATED BY the initial replication process.\n\n";
-	}
 	return 0;
 }
 
@@ -1049,6 +1121,50 @@ sub getsnaps() {
 	return %snaps;
 }
 
+sub getbookmarks() {
+	my ($rhost,$fs,$isroot,%bookmarks) = @_;
+	my $mysudocmd;
+	my $fsescaped = escapeshellparam($fs);
+	if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; }
+
+	if ($rhost ne '') {
+		$rhost = "$sshcmd $rhost";
+		# double escaping needed
+		$fsescaped = escapeshellparam($fsescaped);
+	}
+
+	my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |";
+	if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; }
+	open FH, $getbookmarkcmd;
+	my @rawbookmarks = <FH>;
+	close FH;
+
+	# this is a little obnoxious. get guid,creation returns guid,creation on two separate lines
+	# as though each were an entirely separate get command.
+
+	my $lastguid;
+
+	foreach my $line (@rawbookmarks) {
+		# only import bookmark guids, creation from the specified filesystem
+		if ($line =~ /\Q$fs\E\#.*guid/) {
+			chomp $line;
+			$lastguid = $line;
+			$lastguid =~ s/^.*\tguid\t*(\d*).*/$1/;
+			my $bookmark = $line;
+			$bookmark =~ s/^.*\#(.*)\tguid.*$/$1/;
+			$bookmarks{$lastguid}{'name'}=$bookmark;
+		} elsif ($line =~ /\Q$fs\E\#.*creation/) {
+			chomp $line;
+			my $creation = $line;
+			$creation =~ s/^.*\tcreation\t*(\d*).*/$1/;
+			my $bookmark = $line;
+			$bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/;
+			$bookmarks{$lastguid}{'creation'}=$creation;
+		}
+	}
+
+	return %bookmarks;
+}
 
 sub getsendsize {
 	my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_;

From a1d9e79e70548b88e4f702d3790d9f5e7c028ffc Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 27 Jul 2018 22:58:42 +0200
Subject: [PATCH 20/68] added two tests for zfs bookmark replication

---
 .../run.sh                                    | 56 +++++++++++++++++++
 .../run.sh                                    | 56 +++++++++++++++++++
 tests/syncoid/run-tests.sh                    | 27 +++++++++
 3 files changed, 139 insertions(+)
 create mode 100755 tests/syncoid/1_bookmark_replication_intermediate/run.sh
 create mode 100755 tests/syncoid/2_bookmark_replication_no_intermediate/run.sh
 create mode 100755 tests/syncoid/run-tests.sh

diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/1_bookmark_replication_intermediate/run.sh
new file mode 100755
index 0000000..11edb04
--- /dev/null
+++ b/tests/syncoid/1_bookmark_replication_intermediate/run.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# test replication with fallback to bookmarks and all intermediate snapshots
+
+set -x
+set -e
+
+. ../../common/lib.sh
+
+POOL_IMAGE="/tmp/syncoid-test-1.zpool"
+POOL_SIZE="200M"
+POOL_NAME="syncoid-test-1"
+TARGET_CHECKSUM="a23564d5bb8a2babc3ac8936fd82825ad9fff9c82d4924f5924398106bbda9f0  -"
+
+truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
+
+zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
+
+function cleanUp {
+  zpool export "${POOL_NAME}"
+}
+
+# export pool in any case
+trap cleanUp EXIT
+
+zfs create "${POOL_NAME}"/src
+zfs snapshot "${POOL_NAME}"/src@snap1
+zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1
+# initial replication
+../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
+# destroy last common snapshot on source
+zfs destroy "${POOL_NAME}"/src@snap1
+
+# create intermediate snapshots
+# sleep is needed so creation time can be used for proper sorting
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap2
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap3
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap4
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap5
+
+# replicate which should fallback to bookmarks
+../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
+
+# verify
+output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
+checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum)
+
+if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
+	exit 1
+fi
+
+exit 0
diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh
new file mode 100755
index 0000000..94ac690
--- /dev/null
+++ b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# test replication with fallback to bookmarks and all intermediate snapshots
+
+set -x
+set -e
+
+. ../../common/lib.sh
+
+POOL_IMAGE="/tmp/syncoid-test-2.zpool"
+POOL_SIZE="200M"
+POOL_NAME="syncoid-test-2"
+TARGET_CHECKSUM="2460d4d4417793d2c7a5c72cbea4a8a584c0064bf48d8b6daa8ba55076cba66d  -"
+
+truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
+
+zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
+
+function cleanUp {
+  zpool export "${POOL_NAME}"
+}
+
+# export pool in any case
+trap cleanUp EXIT
+
+zfs create "${POOL_NAME}"/src
+zfs snapshot "${POOL_NAME}"/src@snap1
+zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1
+# initial replication
+../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
+# destroy last common snapshot on source
+zfs destroy "${POOL_NAME}"/src@snap1
+
+# create intermediate snapshots
+# sleep is needed so creation time can be used for proper sorting
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap2
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap3
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap4
+sleep 1
+zfs snapshot "${POOL_NAME}"/src@snap5
+
+# replicate which should fallback to bookmarks
+../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
+
+# verify
+output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name)
+checksum=$(echo "${output}" | sha256sum)
+
+if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
+	exit 1
+fi
+
+exit 0
diff --git a/tests/syncoid/run-tests.sh b/tests/syncoid/run-tests.sh
new file mode 100755
index 0000000..a9843a5
--- /dev/null
+++ b/tests/syncoid/run-tests.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# run's all the available tests
+
+for test in */; do
+    if [ ! -x "${test}/run.sh" ]; then
+        continue
+    fi
+
+    testName="${test%/}"
+
+    LOGFILE=/tmp/syncoid_test_run_"${testName}".log
+
+    pushd . > /dev/null
+
+    echo -n "Running test ${testName} ... "
+    cd "${test}"
+    echo | bash run.sh > "${LOGFILE}" 2>&1
+
+    if [ $? -eq 0 ]; then
+        echo "[PASS]"
+    else
+        echo "[FAILED] (see ${LOGFILE})"
+    fi
+
+    popd > /dev/null
+done

From 089516c58ec43c30342a45abebd60b60b94e41a4 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Sun, 29 Jul 2018 12:43:46 +0200
Subject: [PATCH 21/68] implemented force-delete flag to let syncoid destroy
 target datasets without matching snapshots/bookmarks

---
 syncoid | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/syncoid b/syncoid
index 3cbff6b..8554d43 100755
--- a/syncoid
+++ b/syncoid
@@ -19,7 +19,8 @@ use Sys::Hostname;
 my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
-                   "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2);
+                   "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
+                   "force-delete") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 
@@ -176,7 +177,7 @@ sub getchilddatasets {
 
 sub syncdataset {
 
-	my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_;
+	my ($sourcehost, $sourcefs, $targethost, $targetfs, $skipsnapshot) = @_;
 
 	my $sourcefsescaped = escapeshellparam($sourcefs);
 	my $targetfsescaped = escapeshellparam($targetfs);
@@ -228,7 +229,7 @@ sub syncdataset {
 		    print "\n\n\n";
 		}
 
-		if (!defined $args{'no-sync-snap'}) {
+		if (!defined $args{'no-sync-snap'} && !defined $skipsnapshot) {
 			# create a new syncoid snapshot on the source filesystem.
 			$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
 		} else {
@@ -392,6 +393,30 @@ sub syncdataset {
 			}
 
 			if (! $bookmark) {
+				if ($args{'force-delete'}) {
+					if (!$quiet) { print "Removing $targetfs because no matching snapshots were found\n"; }
+
+					my $rcommand = '';
+					my $mysudocmd = '';
+					my $targetfsescaped = escapeshellparam($targetfs);
+
+					if ($targethost ne '') { $rcommand = "$sshcmd $targethost"; }
+					if (!$targetisroot) { $mysudocmd = $sudocmd; }
+
+					my $prunecmd = "$mysudocmd $zfscmd destroy -r $targetfsescaped; ";
+					if ($targethost ne '') {
+						$prunecmd = escapeshellparam($prunecmd);
+					}
+
+					my $ret = system("$rcommand $prunecmd");
+					if ($ret != 0) {
+						warn "WARNING: $rcommand $prunecmd failed: $?";
+					} else {
+						# redo sync and skip snapshot creation (already taken)
+						return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, 1);
+					}
+				}
+
 				# if we got this far, we failed to find a matching snapshot/bookmark.
 
 				print "\n";

From 8905c003355b27c5bd84ce4da74f834bef85df1c Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Sun, 29 Jul 2018 12:45:12 +0200
Subject: [PATCH 22/68] added test for force-delete flag

---
 tests/syncoid/3_force_delete/run.sh | 47 +++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100755 tests/syncoid/3_force_delete/run.sh

diff --git a/tests/syncoid/3_force_delete/run.sh b/tests/syncoid/3_force_delete/run.sh
new file mode 100755
index 0000000..03ad9fa
--- /dev/null
+++ b/tests/syncoid/3_force_delete/run.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# test replication with deletion of target if no matches are found
+
+set -x
+set -e
+
+. ../../common/lib.sh
+
+POOL_IMAGE="/tmp/syncoid-test-3.zpool"
+POOL_SIZE="200M"
+POOL_NAME="syncoid-test-3"
+TARGET_CHECKSUM="0409a2ac216e69971270817189cef7caa91f6306fad9eab1033955b7e7c6bd4c  -"
+
+truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
+
+zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
+
+function cleanUp {
+  zpool export "${POOL_NAME}"
+}
+
+# export pool in any case
+trap cleanUp EXIT
+
+zfs create "${POOL_NAME}"/src
+zfs create "${POOL_NAME}"/src/1
+zfs create "${POOL_NAME}"/src/2
+zfs create "${POOL_NAME}"/src/3
+
+# initial replication
+../../../syncoid -r --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
+# destroy last common snapshot on source
+zfs destroy "${POOL_NAME}"/src/2@%
+zfs snapshot "${POOL_NAME}"/src/2@test
+sleep 1
+../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1
+
+# verify
+output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name | sed 's/@syncoid_.*$'/@syncoid_/)
+checksum=$(echo "${output}" | sha256sum)
+
+if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then
+	exit 1
+fi
+
+exit 0

From 1cb1209c785a43bd6d162d80ea75dc36f86a99b4 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 30 Jul 2018 09:13:26 +0200
Subject: [PATCH 23/68] documented force-delete flag

---
 README.md | 4 ++++
 syncoid   | 1 +
 2 files changed, 5 insertions(+)

diff --git a/README.md b/README.md
index e359886..4755571 100644
--- a/README.md
+++ b/README.md
@@ -172,6 +172,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to not use resumeable zfs send/receive streams.
 
++ --force-delete
+
+	Remove target datasets recursively (WARNING: this will also affect child datasets with matching snapshots/bookmarks), if there are no matching snapshots/bookmarks.
+
 + --dumpsnaps
 
 	This prints a list of snapshots during the run.
diff --git a/syncoid b/syncoid
index 8554d43..a6500bb 100755
--- a/syncoid
+++ b/syncoid
@@ -1341,3 +1341,4 @@ Options:
   --dumpsnaps           Dumps a list of snapshots during the run
   --no-command-checks   Do not check command existence before attempting transfer. Not recommended
   --no-resume           Don't use the ZFS resume feature if available
+  --force-delete        Remove target datasets recursively, if there are no matching snapshots/bookmarks

From 020f57cb746ad7b5ed9a0fafa0a80b427cf6f3ae Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 30 Jul 2018 09:43:03 +0200
Subject: [PATCH 24/68] check for error while listing snapshots/bookmarks as
 this can be dangerous with force-delete

---
 syncoid | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/syncoid b/syncoid
index a6500bb..26307ba 100755
--- a/syncoid
+++ b/syncoid
@@ -1114,7 +1114,7 @@ sub getsnaps() {
 	if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; }
 	open FH, $getsnapcmd;
 	my @rawsnaps = <FH>;
-	close FH;
+	close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)";
 
 	# this is a little obnoxious. get guid,creation returns guid,creation on two separate lines
 	# as though each were an entirely separate get command.
@@ -1158,11 +1158,21 @@ sub getbookmarks() {
 		$fsescaped = escapeshellparam($fsescaped);
 	}
 
-	my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |";
+	my $error = 0;
+	my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped 2>&1 |";
 	if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; }
 	open FH, $getbookmarkcmd;
 	my @rawbookmarks = <FH>;
-	close FH;
+	close FH or $error = 1;
+
+	if ($error == 1) {
+		if ($rawbookmarks[0] =~ /invalid type/) {
+			# no support for zfs bookmarks, return empty hash
+			return %bookmarks;
+		}
+
+		die "CRITICAL ERROR: bookmarks couldn't be listed for $fs (exit code $?)";
+	}
 
 	# this is a little obnoxious. get guid,creation returns guid,creation on two separate lines
 	# as though each were an entirely separate get command.

From f51bb9db7e44a0d512c29c8ba01b0ad2265d237c Mon Sep 17 00:00:00 2001
From: Martin Schrodt <martin@schrodt.org>
Date: Wed, 15 Aug 2018 16:07:01 +0200
Subject: [PATCH 25/68] add xz compression

---
 syncoid | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/syncoid b/syncoid
index b62eee8..18708d2 100755
--- a/syncoid
+++ b/syncoid
@@ -509,6 +509,12 @@ sub compressargset {
 			decomrawcmd => '/usr/bin/zstd',
 			decomargs   => '-dc',
 		},
+		'xz' => {
+			rawcmd      => '/usr/bin/xz',
+			args        => '',
+			decomrawcmd => '/usr/bin/xz',
+			decomargs   => '-d',
+		},
 		'lzo' => {
 			rawcmd      => '/usr/bin/lzop',
 			args        => '',
@@ -519,7 +525,7 @@ sub compressargset {
 
 	if ($value eq 'default') {
 		$value = $DEFAULT_COMPRESSION;
-	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) {
+	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'xz', 'default', 'none'))) {
 		warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION";
 		$value = $DEFAULT_COMPRESSION;
 	}
@@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool
 
 Options:
 
-  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none
+  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, xz, lzo (default) & none
   --identifier=EXTRA    Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets.
   --recursive|r         Also transfers child datasets
   --skip-parent         Skips syncing of the parent dataset. Does nothing without '--recursive' option.

From 0deaacfc06bdb891525fb1bdc1ed34abcb91b9c1 Mon Sep 17 00:00:00 2001
From: Martin Schrodt <martin@schrodt.org>
Date: Wed, 15 Aug 2018 22:21:56 +0200
Subject: [PATCH 26/68] add xz to README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b833dec..dc1162b 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 + --compress <compression type>
 
-	Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
+	Currently accepted options: gzip, pigz-fast, pigz-slow, xz, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
 
 + --source-bwlimit <limit t|g|m|k>
 

From 8de3cdce212ae15a849768e824d929afd6bc1501 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 3 Sep 2018 17:45:03 +0200
Subject: [PATCH 27/68] let monitor-health also check vdev member io/checksum
 errors

---
 sanoid | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)
 mode change 100755 => 100644 sanoid

diff --git a/sanoid b/sanoid
old mode 100755
new mode 100644
index 485ee08..7ae1b5b
--- a/sanoid
+++ b/sanoid
@@ -974,7 +974,7 @@ sub check_zpool() {
 			}
 
 			## other cases
-			my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/;
+			my ($dev, $sta, $read, $write, $cksum) = /^\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/;
 
 			if (!defined($sta)) {
 				# cache and logs are special and don't have a status
@@ -994,8 +994,21 @@ sub check_zpool() {
 			## no display for verbose level 1
 			next if ($verbose==1);
 			## don't display working devices for verbose level 2
-			next if ($verbose==2 && $state eq "OK");
-			next if ($verbose==2 && ($sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE"));
+			if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")) {
+				# check for io/checksum errors
+
+				my @vdeverr = ();
+				if ($read != 0) { push @vdeverr, "read" };
+				if ($write != 0) { push @vdeverr, "write" };
+				if ($cksum != 0) { push @vdeverr, "cksum" };
+
+				if (scalar @vdeverr) {
+					$dmge=$dmge . "(" . $dev . ":" . join(", ", @vdeverr) . " errors) ";
+					if ($state eq "OK") { $state = "WARNING" };
+				}
+
+				next;
+			}
 
 			## show everything else
 			if (/^\s{3}(\S+)/) {

From 997487d12bf0f7d725199fb9fb9dd5cb4e484504 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 3 Sep 2018 17:46:21 +0200
Subject: [PATCH 28/68] restore filemode

---
 sanoid | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 sanoid

diff --git a/sanoid b/sanoid
old mode 100644
new mode 100755

From f39ed1ec49e989aa4fc7ae8c79f6329f5764cf3b Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 3 Sep 2018 18:32:17 +0200
Subject: [PATCH 29/68] for remote target/source it's required to specify a
 user

---
 syncoid | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/syncoid b/syncoid
index b62eee8..30aef49 100755
--- a/syncoid
+++ b/syncoid
@@ -1218,9 +1218,9 @@ syncoid - ZFS snapshot replication tool
 =head1 SYNOPSIS
 
  syncoid [options]... SOURCE TARGET
- or   syncoid [options]... SOURCE [USER@]HOST:TARGET
- or   syncoid [options]... [USER@]HOST:SOURCE [TARGET]
- or   syncoid [options]... [USER@]HOST:SOURCE [USER@]HOST:TARGET
+ or   syncoid [options]... SOURCE USER@HOST:TARGET
+ or   syncoid [options]... USER@HOST:SOURCE TARGET
+ or   syncoid [options]... USER@HOST:SOURCE USER@HOST:TARGET
 
  SOURCE                Source ZFS dataset. Can be either local or remote
  TARGET                Target ZFS dataset. Can be either local or remote

From 03a074e5c286d77d049df9555cabd3d2cc71d4a2 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Sep 2018 08:13:47 +0200
Subject: [PATCH 30/68] show warning if a key is ignored from a template

---
 sanoid | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/sanoid b/sanoid
index 485ee08..5764849 100755
--- a/sanoid
+++ b/sanoid
@@ -691,10 +691,12 @@ sub init {
 			# override with values from user-defined default template, if any
 
 			foreach my $key (keys %{$ini{'template_default'}}) {
-				if (! ($key =~ /template|recursive/)) {
-				if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; }
-					$config{$section}{$key} = $ini{'template_default'}{$key};
+				if ($key =~ /template|recursive/) {
+					warn "ignored key '$key' from user-defined default template.\n";
+					next;
 				}
+				if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; }
+				$config{$section}{$key} = $ini{'template_default'}{$key};
 			}
 		}
 
@@ -708,10 +710,12 @@ sub init {
 
 				my $template = 'template_'.$rawtemplate;
 				foreach my $key (keys %{$ini{$template}}) {
-					if (! ($key =~ /template|recursive/)) {
-						if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; }
-						$config{$section}{$key} = $ini{$template}{$key};
+					if ($key =~ /template|recursive/) {
+						warn "ignored key '$key' from '$rawtemplate' template.\n";
+						next;
 					}
+					if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; }
+					$config{$section}{$key} = $ini{$template}{$key};
 				}
 			}
 		}

From 65c7be5b1d58287e0fa637c01b6b8265e350fe2a Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Sep 2018 20:17:46 +0200
Subject: [PATCH 31/68] fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b833dec..0354c6b 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ And its /etc/sanoid/sanoid.conf might look something like this:
         autoprune = yes
 ```
 
-Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set).  Except in the case of data/images/win7-spice, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason.
+Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 dailies, 3 monthlies, and no yearlies for all datasets under data/images (but not data/images itself, since process_children_only is set).  Except in the case of data/images/win7, which follows the same template (since it's a child of data/images) but only keeps 4 hourlies for whatever reason.
 
 ##### Sanoid Command Line Options
 

From 2fe97f13ad29ed9f07000475bb0e9aa1430e95f7 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 10:07:05 +0200
Subject: [PATCH 32/68] prepare for v1.4.19 release

---
 CHANGELIST                | 23 +++++++++++++++++++++++
 VERSION                   |  2 +-
 packages/debian/changelog | 27 +++++++++++++++++++++++++++
 packages/rhel/sanoid.spec |  6 ++++--
 sanoid                    |  2 +-
 syncoid                   |  2 +-
 6 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/CHANGELIST b/CHANGELIST
index 515d05d..3ef74ae 100644
--- a/CHANGELIST
+++ b/CHANGELIST
@@ -1,3 +1,26 @@
+1.4.19 [sanoid]  monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
+       [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
+       [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
+       [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0)
+       [syncoid] make local source bwlimit work (@phreaker0)
+       [syncoid] fix 'resume support' detection on FreeBSD (@pit3k)
+       [sanoid]  updated INSTALL with missing dependency
+       [sanoid]  fixed monitor-health command for pools containing cache and log devices (@phreaker0)
+       [sanoid]  quiet flag suppresses all info output (@martinvw)
+       [sanoid]  check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
+       [sanoid]  added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
+       [sanoid]  cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
+       [sanoid]  implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
+       [syncoid] Added support for ZStandard compression.(@danielewood)
+       [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0)
+       [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
+       [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
+       [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0)
+       [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0)
+       [sanoid]  use UTC by default in unit template and documentation (@phreaker0)
+       [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0)
+       [syncoid] documented compatibility issues with (t)csh shells (@ecoutu)
+
 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid,
        thank you @phreaker0!
 
diff --git a/VERSION b/VERSION
index f689e8c..fd4ca57 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.4.18
+1.4.19
diff --git a/packages/debian/changelog b/packages/debian/changelog
index 2bcf423..67cca4d 100644
--- a/packages/debian/changelog
+++ b/packages/debian/changelog
@@ -1,3 +1,30 @@
+sanoid (1.4.19) unstable; urgency=medium
+
+  [sanoid]  monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
+  [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
+  [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
+  [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0)
+  [syncoid] make local source bwlimit work (@phreaker0)
+  [syncoid] fix 'resume support' detection on FreeBSD (@pit3k)
+  [sanoid]  updated INSTALL with missing dependency
+  [sanoid]  fixed monitor-health command for pools containing cache and log devices (@phreaker0)
+  [sanoid]  quiet flag suppresses all info output (@martinvw)
+  [sanoid]  check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
+  [sanoid]  added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
+  [sanoid]  cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
+  [sanoid]  implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
+  [syncoid] Added support for ZStandard compression.(@danielewood)
+  [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0)
+  [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
+  [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
+  [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0)
+  [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0)
+  [sanoid]  use UTC by default in unit template and documentation (@phreaker0)
+  [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0)
+  [syncoid] documented compatibility issues with (t)csh shells (@ecoutu)
+
+ -- Jim Salter <github@jrs-s.net>  Wed, 05 Sep 2018 04:00:00 -0400
+
 sanoid (1.4.18) unstable; urgency=medium
 
   implemented special character handling and support of ZFS resume/receive tokens by default in syncoid,
diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec
index 3a9412f..4971404 100644
--- a/packages/rhel/sanoid.spec
+++ b/packages/rhel/sanoid.spec
@@ -1,4 +1,4 @@
-%global version 1.4.18
+%global version 1.4.19
 %global git_tag v%{version}
 
 # Enable with systemctl "enable sanoid.timer"
@@ -12,7 +12,7 @@ Summary:	   A policy-driven snapshot management tool for ZFS file systems
 Group:		   Applications/System
 License:	   GPLv3
 URL:		   https://github.com/jimsalterjrs/sanoid
-Source0:           https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz
+Source0:	   https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz
 
 Requires:	   perl, mbuffer, lzop, pv
 %if 0%{?_with_systemd}
@@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name}
 %endif
 
 %changelog
+* Wed Sep 05 2018 Christoph Klaffl <christoph@phreaker.eu> - 1.4.19
+- Bump to 1.4.19
 * Sat Apr 28 2018 Dominic Robinson <github@dcrdev.com> - 1.4.18-1
 - Bump to 1.4.18
 * Thu Aug 31 2017 Dominic Robinson <github@dcrdev.com> - 1.4.14-2
diff --git a/sanoid b/sanoid
index 7ae1b5b..fd68145 100755
--- a/sanoid
+++ b/sanoid
@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '1.4.18';
+$::VERSION = '1.4.19';
 
 use strict;
 use warnings;
diff --git a/syncoid b/syncoid
index 30aef49..8aa5205 100755
--- a/syncoid
+++ b/syncoid
@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '1.4.18';
+$::VERSION = '1.4.19';
 
 use strict;
 use warnings;

From 055f26b9709aa412b2597e53c99e00c09cf5f5df Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 18:47:16 +0200
Subject: [PATCH 33/68] ignore unknown interval type to prevent perl warnings

---
 sanoid | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sanoid b/sanoid
index 7ae1b5b..31376ad 100755
--- a/sanoid
+++ b/sanoid
@@ -438,6 +438,9 @@ sub take_snapshots {
 					push @preferredtime,$datestamp{'year'};
 					$lastpreferred = timelocal(@preferredtime);
 					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year
+				} else {
+					# unknown type
+					next;
 				}
 
 				# reconstruct our human-formatted most recent preferred snapshot time into an epoch time, to compare with the epoch of our most recent snapshot

From d5f4d1c121e40da89fcca720ffd55332c3847b04 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 19:01:15 +0200
Subject: [PATCH 34/68] add lz4 compression

---
 README.md |  2 +-
 syncoid   | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b833dec..ce09a37 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 + --compress <compression type>
 
-	Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
+	Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used.
 
 + --source-bwlimit <limit t|g|m|k>
 
diff --git a/syncoid b/syncoid
index 30aef49..c42a008 100755
--- a/syncoid
+++ b/syncoid
@@ -515,11 +515,17 @@ sub compressargset {
 			decomrawcmd => '/usr/bin/lzop',
 			decomargs   => '-dfc',
 		},
+		'lz4' => {
+			rawcmd      => '/usr/bin/lz4',
+			args        => '',
+			decomrawcmd => '/usr/bin/lz4',
+			decomargs   => '-dc',
+		},
 	);
 
 	if ($value eq 'default') {
 		$value = $DEFAULT_COMPRESSION;
-	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lzo', 'default', 'none'))) {
+	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'lzo', 'default', 'none'))) {
 		warn "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION";
 		$value = $DEFAULT_COMPRESSION;
 	}
@@ -1227,7 +1233,7 @@ syncoid - ZFS snapshot replication tool
 
 Options:
 
-  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none
+  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, lzo (default) & none
   --identifier=EXTRA    Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets.
   --recursive|r         Also transfers child datasets
   --skip-parent         Skips syncing of the parent dataset. Does nothing without '--recursive' option.

From 4af294838256fee6cee00d034bd75936992063d3 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 19:18:21 +0200
Subject: [PATCH 35/68] fix uninitialized value warning in debug mode on
 initial run (no snapshots yet)

---
 sanoid | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..0af79a4 100755
--- a/sanoid
+++ b/sanoid
@@ -492,16 +492,20 @@ sub blabber {
 		my $path = $config{$section}{'path'};
 		print "Filesystem $path has:\n";
 		print "     $snapsbypath{$path}{'numsnaps'} total snapshots ";
-		print "(newest: ";
-		my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60);
-		print "$newest hours old)\n";
+		if ($snapsbypath{$path}{'numsnaps'} == 0) {
+			print "(no current snapshots)"
+		} else {
+			print "(newest: ";
+			my $newest = sprintf("%.1f",$snapsbypath{$path}{'newest'} / 60 / 60);
+			print "$newest hours old)\n";
 
-		foreach my $type (keys %{ $snapsbytype{$path} }){
-			print "          $snapsbytype{$path}{$type}{'numsnaps'} $type\n";
-			print "              desired: $config{$section}{$type}\n";
-			print "              newest: ";
-			my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60));
-			print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n";
+			foreach my $type (keys %{ $snapsbytype{$path} }){
+				print "          $snapsbytype{$path}{$type}{'numsnaps'} $type\n";
+				print "              desired: $config{$section}{$type}\n";
+				print "              newest: ";
+				my $newest = sprintf("%.1f",($snapsbytype{$path}{$type}{'newest'} / 60 / 60));
+				print "$newest hours old, named $snapsbytype{$path}{$type}{'newestname'}\n";
+			}
 		}
 		print "\n\n";
 	}

From 253b8b467d2de2962826842052645d4439c0fd40 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 19:34:44 +0200
Subject: [PATCH 36/68] parse values for recursive key like the others
 booleans, previously any non empty value was considered as true

---
 sanoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..e5d3e05 100755
--- a/sanoid
+++ b/sanoid
@@ -743,7 +743,7 @@ sub init {
 
 		# how 'bout some recursion? =)
 		my @datasets;
-		if ($ini{$section}{'recursive'}) {
+		if (grep( /^$ini{$section}{'recursive'}$/, @istrue )) {
 			@datasets = getchilddatasets($config{$section}{'path'});
 			foreach my $dataset(@datasets) {
 				chomp $dataset;

From 807fc53afb4d28b3742af506d8e3f9b0ee2a4337 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 5 Sep 2018 22:46:14 +0200
Subject: [PATCH 37/68] added all available syncoid/sanoid parameters to README

---
 README.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/README.md b/README.md
index b833dec..9fcfcf6 100644
--- a/README.md
+++ b/README.md
@@ -88,6 +88,13 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da
 
 	This prints out quite alot of additional information during a sanoid run, and is normally not needed.
 
++ --readonly
+
+	Skip creation/deletion of snapshots (Simulate).
+
++ --help
+
+	Show help message.
 
 ----------
 
@@ -206,6 +213,14 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	Allow sync to/from boxes running SSH on non-standard ports.
 
++ --sshcipher
+
+	Instruct ssh to use a particular cipher set.
+
++ --sshoption
+
+	Passes option to ssh. This argument can be specified multiple times.
+
 + --sshkey
 
 	Use specified identity file as per ssh -i.
@@ -218,6 +233,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This prints out quite alot of additional information during a sanoid run, and is normally not needed.
 
++ --help
+
+	Show help message.
+
 + --version
 
 	Print the version and exit.

From e7bd567acb1aed210f7d36cd9933460cab080da2 Mon Sep 17 00:00:00 2001
From: Andrew DeMaria <lostonamountain@gmail.com>
Date: Mon, 10 Sep 2018 22:14:05 -0400
Subject: [PATCH 38/68] Do not monitor snapshots types that are set to 0.

Signed-off-by: Andrew DeMaria <lostonamountain@gmail.com>
---
 sanoid | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sanoid b/sanoid
index 7ae1b5b..aa6369d 100755
--- a/sanoid
+++ b/sanoid
@@ -127,6 +127,7 @@ sub monitor_snapshots {
 
 		my @types = ('yearly','monthly','daily','hourly');
 		foreach my $type (@types) {
+			if ($config{$section}{$type} == 0) { next; }
 
 			my $smallerperiod = 0;
 			# we need to set the period length in seconds first

From 108fe5e2fc1d31a1f74e3dfa0448814de3af6ee3 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Wed, 26 Sep 2018 14:02:19 +0200
Subject: [PATCH 39/68] Reversed zfsisbusy and force-prune checks

---
 sanoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanoid b/sanoid
index fcbb630..32651ab 100755
--- a/sanoid
+++ b/sanoid
@@ -293,7 +293,7 @@ sub prune_snapshots {
 						writelock('sanoid_pruning');
 						foreach my $snap( @prunesnaps ){
 							if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
-							if (iszfsbusy($path) && !$args{'force-prune'}) {
+							if (!$args{'force-prune'} && iszfsbusy($path)) {
 								if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; }
 							} else {
 								if (! $args{'readonly'}) {

From f04be06f392911a2044d893065ca52b3e7cf99ea Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Wed, 26 Sep 2018 14:24:38 +0200
Subject: [PATCH 40/68] Fixed indentation

---
 sanoid  | 2 +-
 syncoid | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sanoid b/sanoid
index 32651ab..7a5fdee 100755
--- a/sanoid
+++ b/sanoid
@@ -1323,7 +1323,7 @@ Options:
   --monitor-snapshots   Reports on snapshot "health", in a Nagios compatible format
   --take-snapshots      Creates snapshots as specified in sanoid.conf
   --prune-snapshots     Purges expired snapshots as specified in sanoid.conf
-  --force-prune		Purges expired snapshots even if a send/recv is in progress
+  --force-prune         Purges expired snapshots even if a send/recv is in progress
 
   --help                Prints this helptext
   --version             Prints the version number
diff --git a/syncoid b/syncoid
index f2335dc..8e67f5e 100755
--- a/syncoid
+++ b/syncoid
@@ -1246,7 +1246,7 @@ Options:
   --target-bwlimit=<limit k|m|g|t>  Bandwidth limit on the target transfer
   --no-stream           Replicates using newest snapshot instead of intermediates
   --no-sync-snap        Does not create new snapshot, only transfers existing
-  --no-clone-rollback	Does not rollback clones on target
+  --no-clone-rollback   Does not rollback clones on target
   --no-rollback         Does not rollback clones or snapshots on target (it probably requires a readonly target)
   --exclude=REGEX       Exclude specific datasets which match the given regular expression. Can be specified multiple times
   --sshkey=FILE         Specifies a ssh public key to use to connect

From e9a330f89a8ed9894df7d1a7bf6a996bcec3918f Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 5 Oct 2018 16:19:55 +0200
Subject: [PATCH 41/68] adapt test as it works correctly know after the DST
 patch

---
 tests/1_one_year/run.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh
index 7cec813..1cae7b4 100755
--- a/tests/1_one_year/run.sh
+++ b/tests/1_one_year/run.sh
@@ -10,7 +10,7 @@ set -x
 POOL_NAME="sanoid-test-1"
 POOL_TARGET=""  # root
 RESULT="/tmp/sanoid_test_result"
-RESULT_CHECKSUM="aa15e5595b0ed959313289ecb70323dad9903328ac46e881da5c4b0f871dd7cf"
+RESULT_CHECKSUM="68c67161a59d0e248094a66061972f53613067c9db52ad981030f36bc081fed7"
 
 # UTC timestamp of start and end
 START="1483225200"
@@ -46,10 +46,4 @@ done
 saveSnapshotList "${POOL_NAME}" "${RESULT}"
 
 # hourly daily monthly
-verifySnapshotList "${RESULT}" 8759 366 12 "${RESULT_CHECKSUM}"
-
-# hourly count should be 8760 but one hour get's lost because of DST
-
-# daily count should be 365 but one additional daily is taken
-# because the DST change leads to a day with 25 hours
-# which will trigger an additional daily snapshot
+verifySnapshotList "${RESULT}" 8760 365 12 "${RESULT_CHECKSUM}"

From e83ec060fb68ad47545e24cde71e60c3d63f2752 Mon Sep 17 00:00:00 2001
From: Michael Bushey <git@sendthemtomir.com>
Date: Sun, 14 Oct 2018 15:10:25 -0700
Subject: [PATCH 42/68] INSTALL: Fix name p5-Config-Inifiles ->
 p5-Config-IniFiles

---
 INSTALL | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/INSTALL b/INSTALL
index f0de17b..15c4896 100644
--- a/INSTALL
+++ b/INSTALL
@@ -30,4 +30,4 @@ strongly recommends using your distribution's repositories instead.
 
 On Ubuntu: apt install libconfig-inifiles-perl
 On CentOS: yum install perl-Config-IniFiles
-On FreeBSD: pkg install p5-Config-Inifiles
+On FreeBSD: pkg install p5-Config-IniFiles

From c6ffbf5c4c05e8805a441656b253836631272221 Mon Sep 17 00:00:00 2001
From: Julien Riou <julien.riou@outlook.com>
Date: Sat, 13 Jan 2018 14:26:26 +0100
Subject: [PATCH 43/68] Add pre and post snapshot scripts

---
 sanoid               | 25 ++++++++++++++++++++++++-
 sanoid.conf          | 11 +++++++++++
 sanoid.defaults.conf |  4 ++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..e45133b 100755
--- a/sanoid
+++ b/sanoid
@@ -455,6 +455,18 @@ sub take_snapshots {
 
 	if ( (scalar(@newsnaps)) > 0) {
 		foreach my $snap ( @newsnaps ) {
+			my $dataset = (split '@', $snap)[0];
+			my $presnapshotfailure = 0;
+			if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) {
+				$ENV{'SANOID_TARGET'} = $dataset;
+				if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; }
+				if (system($config{$dataset}{'pre_snapshot_script'}) != 0) {
+					warn "WARN: pre_snapshot_script failed, $?";
+					$config{$dataset}{'no_inconsistent_snapshot'} and next;
+					$presnapshotfailure = 1;
+				}
+				delete $ENV{'SANOID_TARGET'};
+			}
 			if ($args{'verbose'}) { print "taking snapshot $snap\n"; }
 			if (!$args{'readonly'}) {
 				system($zfs, "snapshot", "$snap") == 0
@@ -462,6 +474,17 @@ sub take_snapshots {
 				# make sure we don't end up with multiple snapshots with the same ctime
 				sleep 1;
 			}
+			if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) {
+				if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) {
+					$ENV{'SANOID_TARGET'} = $dataset;
+					if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; }
+					if (system($config{$dataset}{'post_snapshot_script'}) != 0) {
+						warn "WARN: post_snapshot_script failed, $?";
+						$config{$dataset}{'no_inconsistent_snapshot'} and next;
+					}
+					delete $ENV{'SANOID_TARGET'};
+				}
+			}
 		}
 		$forcecacheupdate = 1;
 		%snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate);
@@ -661,7 +684,7 @@ sub init {
 	tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!";
 
 	# we'll use these later to normalize potentially true and false values on any toggle keys
-	my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only');
+	my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','no_inconsistent_snapshot','force_post_snapshot_script');
 	my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
 	my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
 
diff --git a/sanoid.conf b/sanoid.conf
index 9b1f19d..218a492 100644
--- a/sanoid.conf
+++ b/sanoid.conf
@@ -67,6 +67,17 @@
 	daily_warn = 48
 	daily_crit = 60
 
+[template_scripts]
+	### run script before snapshot
+	### dataset name will be supplied as an environment variable $SANOID_TARGET
+	pre_snapshot_script = /path/to/script.sh
+	### run script after snapshot
+	### dataset name will be supplied as an environment variable $SANOID_TARGET
+	post_snapshot_script = /path/to/script.sh
+	### don't take an inconsistent snapshot
+	#no_inconsistent_snapshot = yes
+	### run post_snapshot_script when pre_snapshot_script is failing
+	#force_post_snapshot_script = yes
 
 [template_ignore]
 	autoprune = no
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index d86cc47..12a8049 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -15,6 +15,10 @@ path =
 recursive =
 use_template =
 process_children_only =
+pre_snapshot_script =
+post_snapshot_script =
+no_inconsistent_snapshot =
+force_post_snapshot_script =
 
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
 # prune any of those type snapshots already present.

From 84213216ec2d98182e31c7e43ecce48cd353404a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niccol=C3=B2=20Belli?= <niccolo.belli@linuxsystems.it>
Date: Tue, 2 Oct 2018 00:47:25 +0200
Subject: [PATCH 44/68] Expose snapshot name through ENV variable

---
 sanoid | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sanoid b/sanoid
index e45133b..73efd5e 100755
--- a/sanoid
+++ b/sanoid
@@ -456,9 +456,11 @@ sub take_snapshots {
 	if ( (scalar(@newsnaps)) > 0) {
 		foreach my $snap ( @newsnaps ) {
 			my $dataset = (split '@', $snap)[0];
+			my $snapname = (split '@', $snap)[1];
 			my $presnapshotfailure = 0;
 			if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) {
 				$ENV{'SANOID_TARGET'} = $dataset;
+				$ENV{'SANOID_SNAPNAME'} = $snapname;
 				if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; }
 				if (system($config{$dataset}{'pre_snapshot_script'}) != 0) {
 					warn "WARN: pre_snapshot_script failed, $?";
@@ -466,6 +468,7 @@ sub take_snapshots {
 					$presnapshotfailure = 1;
 				}
 				delete $ENV{'SANOID_TARGET'};
+				delete $ENV{'SANOID_SNAPNAME'};
 			}
 			if ($args{'verbose'}) { print "taking snapshot $snap\n"; }
 			if (!$args{'readonly'}) {
@@ -477,12 +480,14 @@ sub take_snapshots {
 			if ($config{$dataset}{'post_snapshot_script'} and !$args{'readonly'}) {
 				if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) {
 					$ENV{'SANOID_TARGET'} = $dataset;
+					$ENV{'SANOID_SNAPNAME'} = $snapname;
 					if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; }
 					if (system($config{$dataset}{'post_snapshot_script'}) != 0) {
 						warn "WARN: post_snapshot_script failed, $?";
 						$config{$dataset}{'no_inconsistent_snapshot'} and next;
 					}
 					delete $ENV{'SANOID_TARGET'};
+					delete $ENV{'SANOID_SNAPNAME'};
 				}
 			}
 		}

From fb6608bf47a9508fb3b661a364e597b94b517e5f Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 16 Oct 2018 10:48:04 +0200
Subject: [PATCH 45/68] implemented timeout for pre/post script execution and
 made sure environment is cleaned up after script failure

---
 sanoid | 50 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/sanoid b/sanoid
index 73efd5e..c12dff8 100755
--- a/sanoid
+++ b/sanoid
@@ -458,17 +458,21 @@ sub take_snapshots {
 			my $dataset = (split '@', $snap)[0];
 			my $snapname = (split '@', $snap)[1];
 			my $presnapshotfailure = 0;
+			my $timeout = 5;
 			if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) {
 				$ENV{'SANOID_TARGET'} = $dataset;
 				$ENV{'SANOID_SNAPNAME'} = $snapname;
 				if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; }
-				if (system($config{$dataset}{'pre_snapshot_script'}) != 0) {
-					warn "WARN: pre_snapshot_script failed, $?";
+				my $ret = runscript('pre_snapshot_script',$dataset,$timeout);
+
+				delete $ENV{'SANOID_TARGET'};
+				delete $ENV{'SANOID_SNAPNAME'};
+
+				if ($ret != 0) {
+					# warning was already thrown by runscript function
 					$config{$dataset}{'no_inconsistent_snapshot'} and next;
 					$presnapshotfailure = 1;
 				}
-				delete $ENV{'SANOID_TARGET'};
-				delete $ENV{'SANOID_SNAPNAME'};
 			}
 			if ($args{'verbose'}) { print "taking snapshot $snap\n"; }
 			if (!$args{'readonly'}) {
@@ -482,10 +486,8 @@ sub take_snapshots {
 					$ENV{'SANOID_TARGET'} = $dataset;
 					$ENV{'SANOID_SNAPNAME'} = $snapname;
 					if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; }
-					if (system($config{$dataset}{'post_snapshot_script'}) != 0) {
-						warn "WARN: post_snapshot_script failed, $?";
-						$config{$dataset}{'no_inconsistent_snapshot'} and next;
-					}
+					runscript('post_snapshot_script',$dataset,$timeout);
+
 					delete $ENV{'SANOID_TARGET'};
 					delete $ENV{'SANOID_SNAPNAME'};
 				}
@@ -1337,6 +1339,38 @@ sub removecachedsnapshots {
 	undef %pruned;
 }
 
+#######################################################################################################################3
+#######################################################################################################################3
+#######################################################################################################################3
+
+sub runscript {
+	my $key=shift;
+	my $dataset=shift;
+	my $timeout=shift;
+
+	my $ret;
+	eval {
+		local $SIG{ALRM} = sub { die "alarm\n" };
+		alarm $timeout;
+		$ret = system($config{$dataset}{$key});
+		alarm 0;
+	};
+	if ($@) {
+		if ($@ eq "alarm\n") {
+			warn "WARN: $key didn't finish in the allowed time!";
+		} else {
+			warn "CRITICAL ERROR: $@";
+		}
+		return -1;
+	} else {
+		if ($ret != 0) {
+			warn "WARN: $key failed, $?";
+		}
+	}
+
+	return $ret;
+}
+
 __END__
 
 =head1 NAME

From 0a7fdcb232d5e75eed388f016060cd4736c6185a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niccol=C3=B2=20Belli?= <niccolo.belli@linuxsystems.it>
Date: Sun, 14 Oct 2018 16:28:24 +0200
Subject: [PATCH 46/68] Add pruning hooks

---
 sanoid               | 11 +++++++++++
 sanoid.conf          |  2 ++
 sanoid.defaults.conf |  1 +
 3 files changed, 14 insertions(+)

diff --git a/sanoid b/sanoid
index c12dff8..866eef0 100755
--- a/sanoid
+++ b/sanoid
@@ -299,6 +299,17 @@ sub prune_snapshots {
 								if (! $args{'readonly'}) {
 									if (system($zfs, "destroy", $snap) == 0) {
 										$pruned{$snap} = 1;
+										my $dataset = (split '@', $snap)[0];
+										my $snapname = (split '@', $snap)[1];
+										if ($config{$dataset}{'pruning_script'}) {
+											$ENV{'SANOID_TARGET'} = $dataset;
+											$ENV{'SANOID_SNAPNAME'} = $snapname;
+											if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; }
+											system($config{$dataset}{'pruning_script'}) == 0
+												or warn "WARN: pruning_script failed, $?";
+											delete $ENV{'SANOID_TARGET'};
+											delete $ENV{'SANOID_SNAPNAME'};
+										}
 									} else {
 										warn "could not remove $snap : $?";
 									}
diff --git a/sanoid.conf b/sanoid.conf
index 218a492..e684614 100644
--- a/sanoid.conf
+++ b/sanoid.conf
@@ -78,6 +78,8 @@
 	#no_inconsistent_snapshot = yes
 	### run post_snapshot_script when pre_snapshot_script is failing
 	#force_post_snapshot_script = yes
+	### dataset name will be supplied as an environment variable $SANOID_TARGET
+	pruning_script = /path/to/script.sh
 
 [template_ignore]
 	autoprune = no
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 12a8049..d4dd19e 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -19,6 +19,7 @@ pre_snapshot_script =
 post_snapshot_script =
 no_inconsistent_snapshot =
 force_post_snapshot_script =
+pruning_script =
 
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
 # prune any of those type snapshots already present.

From a7b7fe8d15adc826d78f914f962b0b6929df88cb Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 16 Oct 2018 11:58:25 +0200
Subject: [PATCH 47/68] let pruning script timeout so it doesn't hang sanoid

---
 sanoid | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sanoid b/sanoid
index 866eef0..7c213e4 100755
--- a/sanoid
+++ b/sanoid
@@ -302,11 +302,12 @@ sub prune_snapshots {
 										my $dataset = (split '@', $snap)[0];
 										my $snapname = (split '@', $snap)[1];
 										if ($config{$dataset}{'pruning_script'}) {
+											my $timeout = 5;
 											$ENV{'SANOID_TARGET'} = $dataset;
 											$ENV{'SANOID_SNAPNAME'} = $snapname;
 											if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; }
-											system($config{$dataset}{'pruning_script'}) == 0
-												or warn "WARN: pruning_script failed, $?";
+											my $ret = runscript('pruning_script',$dataset,$timeout);
+
 											delete $ENV{'SANOID_TARGET'};
 											delete $ENV{'SANOID_SNAPNAME'};
 										}

From a8d5c5652a82c505b117ccd02a39962be240bd10 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 16 Oct 2018 17:54:37 +0200
Subject: [PATCH 48/68] make script timeout configureable

---
 sanoid               | 16 +++++++++-------
 sanoid.conf          |  6 ++++--
 sanoid.defaults.conf |  3 ++-
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/sanoid b/sanoid
index 7c213e4..9c0e54d 100755
--- a/sanoid
+++ b/sanoid
@@ -302,11 +302,10 @@ sub prune_snapshots {
 										my $dataset = (split '@', $snap)[0];
 										my $snapname = (split '@', $snap)[1];
 										if ($config{$dataset}{'pruning_script'}) {
-											my $timeout = 5;
 											$ENV{'SANOID_TARGET'} = $dataset;
 											$ENV{'SANOID_SNAPNAME'} = $snapname;
 											if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; }
-											my $ret = runscript('pruning_script',$dataset,$timeout);
+											my $ret = runscript('pruning_script',$dataset);
 
 											delete $ENV{'SANOID_TARGET'};
 											delete $ENV{'SANOID_SNAPNAME'};
@@ -475,7 +474,7 @@ sub take_snapshots {
 				$ENV{'SANOID_TARGET'} = $dataset;
 				$ENV{'SANOID_SNAPNAME'} = $snapname;
 				if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; }
-				my $ret = runscript('pre_snapshot_script',$dataset,$timeout);
+				my $ret = runscript('pre_snapshot_script',$dataset);
 
 				delete $ENV{'SANOID_TARGET'};
 				delete $ENV{'SANOID_SNAPNAME'};
@@ -498,7 +497,7 @@ sub take_snapshots {
 					$ENV{'SANOID_TARGET'} = $dataset;
 					$ENV{'SANOID_SNAPNAME'} = $snapname;
 					if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; }
-					runscript('post_snapshot_script',$dataset,$timeout);
+					runscript('post_snapshot_script',$dataset);
 
 					delete $ENV{'SANOID_TARGET'};
 					delete $ENV{'SANOID_SNAPNAME'};
@@ -1358,12 +1357,15 @@ sub removecachedsnapshots {
 sub runscript {
 	my $key=shift;
 	my $dataset=shift;
-	my $timeout=shift;
+
+	my $timeout=$config{$dataset}{'script_timeout'};
 
 	my $ret;
 	eval {
-		local $SIG{ALRM} = sub { die "alarm\n" };
-		alarm $timeout;
+		if ($timeout gt 0) {
+			local $SIG{ALRM} = sub { die "alarm\n" };
+			alarm $timeout;
+		}
 		$ret = system($config{$dataset}{$key});
 		alarm 0;
 	};
diff --git a/sanoid.conf b/sanoid.conf
index e684614..db468e2 100644
--- a/sanoid.conf
+++ b/sanoid.conf
@@ -74,12 +74,14 @@
 	### run script after snapshot
 	### dataset name will be supplied as an environment variable $SANOID_TARGET
 	post_snapshot_script = /path/to/script.sh
+	### dataset name will be supplied as an environment variable $SANOID_TARGET
+	pruning_script = /path/to/script.sh
 	### don't take an inconsistent snapshot
 	#no_inconsistent_snapshot = yes
 	### run post_snapshot_script when pre_snapshot_script is failing
 	#force_post_snapshot_script = yes
-	### dataset name will be supplied as an environment variable $SANOID_TARGET
-	pruning_script = /path/to/script.sh
+	### limit allowed execution time of scripts before continuing (<= 0 -> infinite)
+	script_timeout = 5
 
 [template_ignore]
 	autoprune = no
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index d4dd19e..d8e428a 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -17,9 +17,10 @@ use_template =
 process_children_only =
 pre_snapshot_script =
 post_snapshot_script =
+pruning_script =
+script_timeout = 5
 no_inconsistent_snapshot =
 force_post_snapshot_script =
-pruning_script =
 
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
 # prune any of those type snapshots already present.

From 6968e441468355a03bd1d6fdd39c5c93d2409d36 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 16 Oct 2018 18:10:57 +0200
Subject: [PATCH 49/68] updated documentation regarding pre/post/prun scripts

---
 sanoid      |  1 -
 sanoid.conf | 10 +++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/sanoid b/sanoid
index 9c0e54d..69562f7 100755
--- a/sanoid
+++ b/sanoid
@@ -469,7 +469,6 @@ sub take_snapshots {
 			my $dataset = (split '@', $snap)[0];
 			my $snapname = (split '@', $snap)[1];
 			my $presnapshotfailure = 0;
-			my $timeout = 5;
 			if ($config{$dataset}{'pre_snapshot_script'} and !$args{'readonly'}) {
 				$ENV{'SANOID_TARGET'} = $dataset;
 				$ENV{'SANOID_SNAPNAME'} = $snapname;
diff --git a/sanoid.conf b/sanoid.conf
index db468e2..feb2237 100644
--- a/sanoid.conf
+++ b/sanoid.conf
@@ -68,19 +68,19 @@
 	daily_crit = 60
 
 [template_scripts]
+	### dataset and snapshot name will be supplied as environment variables
+	### for all pre/post/prune scripts ($SANOID_TARGET, $SANOID_SNAPNAME)
 	### run script before snapshot
-	### dataset name will be supplied as an environment variable $SANOID_TARGET
 	pre_snapshot_script = /path/to/script.sh
 	### run script after snapshot
-	### dataset name will be supplied as an environment variable $SANOID_TARGET
 	post_snapshot_script = /path/to/script.sh
-	### dataset name will be supplied as an environment variable $SANOID_TARGET
+	### run script after pruning snapshot
 	pruning_script = /path/to/script.sh
-	### don't take an inconsistent snapshot
+	### don't take an inconsistent snapshot (skip if pre script fails)
 	#no_inconsistent_snapshot = yes
 	### run post_snapshot_script when pre_snapshot_script is failing
 	#force_post_snapshot_script = yes
-	### limit allowed execution time of scripts before continuing (<= 0 -> infinite)
+	### limit allowed execution time of scripts before continuing (<= 0: infinite)
 	script_timeout = 5
 
 [template_ignore]

From 8e929a331dda6a00ab77208745b7642a2fab7d43 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 9 Nov 2018 07:31:51 +0100
Subject: [PATCH 50/68] removed sleeping between snapshot taking

---
 sanoid | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..57acc7b 100755
--- a/sanoid
+++ b/sanoid
@@ -459,8 +459,6 @@ sub take_snapshots {
 			if (!$args{'readonly'}) {
 				system($zfs, "snapshot", "$snap") == 0
 					or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?";
-				# make sure we don't end up with multiple snapshots with the same ctime
-				sleep 1;
 			}
 		}
 		$forcecacheupdate = 1;

From fa3c511dc106008bd03066059ed819f231734438 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 14 Nov 2018 18:26:11 +0100
Subject: [PATCH 51/68] split snapshot taking/pruning into seperate units for
 debian package to prevent pruning blocking snapshot taking

---
 packages/debian/rules                | 12 +++++++++++-
 packages/debian/sanoid-prune.service | 13 +++++++++++++
 packages/debian/sanoid.service       |  2 +-
 3 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 packages/debian/sanoid-prune.service

diff --git a/packages/debian/rules b/packages/debian/rules
index 83eb475..ddd77b0 100755
--- a/packages/debian/rules
+++ b/packages/debian/rules
@@ -16,4 +16,14 @@ override_dh_auto_install:
 	@mkdir -p $(DESTDIR)/usr/share/doc/sanoid; \
 		cp sanoid.conf $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example;
 	@mkdir -p $(DESTDIR)/lib/systemd/system; \
-		cp debian/sanoid.timer $(DESTDIR)/lib/systemd/system;
+		cp debian/sanoid-prune.service $(DESTDIR)/lib/systemd/system;
+
+override_dh_installinit:
+	dh_installinit --noscripts
+
+override_dh_systemd_enable:
+	dh_systemd_enable sanoid.timer
+	dh_systemd_enable sanoid-prune.service
+
+override_dh_systemd_start:
+	dh_systemd_start sanoid.timer
diff --git a/packages/debian/sanoid-prune.service b/packages/debian/sanoid-prune.service
new file mode 100644
index 0000000..c956bd5
--- /dev/null
+++ b/packages/debian/sanoid-prune.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Cleanup ZFS Pool
+Requires=zfs.target
+After=zfs.target sanoid.service
+ConditionFileNotEmpty=/etc/sanoid/sanoid.conf
+
+[Service]
+Environment=TZ=UTC
+Type=oneshot
+ExecStart=/usr/sbin/sanoid --prune-snapshots
+
+[Install]
+WantedBy=sanoid.service
diff --git a/packages/debian/sanoid.service b/packages/debian/sanoid.service
index 2d01bbf..e146354 100644
--- a/packages/debian/sanoid.service
+++ b/packages/debian/sanoid.service
@@ -7,4 +7,4 @@ ConditionFileNotEmpty=/etc/sanoid/sanoid.conf
 [Service]
 Environment=TZ=UTC
 Type=oneshot
-ExecStart=/usr/sbin/sanoid --cron
+ExecStart=/usr/sbin/sanoid --take-snapshots

From 2796e22dbf8a6eaea8b4d1d27b04f9bea4636ddb Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 21 Nov 2018 00:34:21 +0100
Subject: [PATCH 52/68] added option to defer pruning based on the available
 pool capacity

---
 sanoid               | 68 +++++++++++++++++++++++++++++++++++++++++---
 sanoid.defaults.conf |  4 ++-
 2 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..34445aa 100755
--- a/sanoid
+++ b/sanoid
@@ -31,6 +31,7 @@ if (keys %args < 2) {
 my $pscmd = '/bin/ps';
 
 my $zfs = '/sbin/zfs';
+my $zpool = '/sbin/zpool';
 
 my $conf_file = "$args{'configdir'}/sanoid.conf";
 my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf";
@@ -44,6 +45,7 @@ my $cache = '/var/cache/sanoidsnapshots.txt';
 my $cacheTTL = 900; # 15 minutes
 my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate );
 my %pruned;
+my %capacitycache;
 
 my %snapsbytype = getsnapsbytype( \%config, \%snaps );
 
@@ -254,6 +256,10 @@ sub prune_snapshots {
 		my $path = $config{$section}{'path'};
 
 		my $period = 0;
+		if (check_prune_defer($config, $section)) {
+			if ($args{'verbose'}) { print "INFO: deferring snapshot pruning ($section)...\n"; }
+			next;
+		}
 
 		foreach my $type (keys %{ $config{$section} }){
 			unless ($type =~ /ly$/) { next; }
@@ -872,7 +878,7 @@ sub check_zpool() {
 		exit $ERRORS{$state};
 	}
 
-	my $statcommand="/sbin/zpool list -o name,size,cap,health,free $pool";
+	my $statcommand="$zpool list -o name,size,cap,health,free $pool";
 
 	if (! open STAT, "$statcommand|") {
 		print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n");
@@ -920,7 +926,7 @@ sub check_zpool() {
 	## flag to detect section of zpool status involving our zpool
 	my $poolfind=0;
 
-	$statcommand="/sbin/zpool status $pool";
+	$statcommand="$zpool status $pool";
 	if (! open STAT, "$statcommand|") {
 		$state = 'CRITICAL';
 		print ("$state '$statcommand' command returns no result! NOTE: This plugin needs OS support for ZFS, and execution with root privileges.\n");
@@ -1028,7 +1034,7 @@ sub check_zpool() {
 	return ($ERRORS{$state},$msg);
 } # end check_zpool()
 
-sub check_capacity_limit() {
+sub check_capacity_limit {
 	my $value = shift;
 
 	if (!defined($value) || $value !~ /^\d+\z/) {
@@ -1051,7 +1057,7 @@ sub check_zpool_capacity() {
 	my $capacitylimitsref=shift;
 	my %capacitylimits=%$capacitylimitsref;
 
-	my $statcommand="/sbin/zpool list -H -o cap $pool";
+	my $statcommand="$zpool list -H -o cap $pool";
 
 	if (! open STAT, "$statcommand|") {
 		print ("$state '$statcommand' command returns no result!\n");
@@ -1096,6 +1102,60 @@ sub check_zpool_capacity() {
 	return ($ERRORS{$state},$msg);
 } # end check_zpool_capacity()
 
+sub check_prune_defer {
+	my ($config, $section) = @_;
+
+	my $limit = $config{$section}{"prune_defer"};
+
+	if (!check_capacity_limit($limit)) {
+		die "ERROR: invalid prune_defer limit!\n";
+	}
+
+	if ($limit eq 0) {
+		return 0;
+	}
+
+	my @parts = split /\//, $section, 2;
+	my $pool = $parts[0];
+
+	if (exists $capacitycache{$pool}) {
+	} else {
+		$capacitycache{$pool} = get_zpool_capacity($pool);
+	}
+
+	if ($limit < $capacitycache{$pool}) {
+		return 0;
+	}
+
+	return 1;
+}
+
+sub get_zpool_capacity {
+	my $pool = shift;
+
+	my $statcommand="$zpool list -H -o cap $pool";
+
+	if (! open STAT, "$statcommand|") {
+		die "ERROR: '$statcommand' command returns no result!\n";
+	}
+
+	my $line = <STAT>;
+	close(STAT);
+
+	chomp $line;
+	my @row = split(/ +/, $line);
+	my $cap=$row[0];
+
+	## check for valid capacity value
+	if ($cap !~ m/^[0-9]{1,3}%$/ ) {
+		die "ERROR: '$statcommand' command returned invalid capacity value ($cap)!\n";
+	}
+
+	$cap =~ s/\D//g;
+
+	return $cap;
+}
+
 ######################################################################################################
 ######################################################################################################
 ######################################################################################################
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index d86cc47..0797fa8 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -26,7 +26,9 @@ hourly = 48
 daily = 90
 monthly = 6
 yearly = 0
-min_percent_free = 10
+# pruning can be skipped based on the used capacity of the pool
+# (0: always prune, 1-100: only prune if used capacity is greater than this value)
+prune_defer = 0
 
 # We will automatically take snapshots if autosnap is on, at the desired times configured
 # below (or immediately, if we don't have one since the last preferred time for that type).

From c8b880c5e2ebb4771fa845af8e4cd006751d8970 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 21 Nov 2018 18:01:40 +0100
Subject: [PATCH 53/68] implemented clone handling (try to recreate on target
 instead of full replication)

---
 syncoid | 116 +++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 93 insertions(+), 23 deletions(-)

diff --git a/syncoid b/syncoid
index 30aef49..e9e3f27 100755
--- a/syncoid
+++ b/syncoid
@@ -104,17 +104,59 @@ my $exitcode = 0;
 ## replication                                            ##
 
 if (!defined $args{'recursive'}) {
-	syncdataset($sourcehost, $sourcefs, $targethost, $targetfs);
+	syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef);
 } else {
 	if ($debug) { print "DEBUG: recursive sync of $sourcefs.\n"; }
 	my @datasets = getchilddatasets($sourcehost, $sourcefs, $sourceisroot);
-	foreach my $dataset(@datasets) {
+
+	my @deferred;
+
+	foreach my $datasetProperties(@datasets) {
+		my $dataset = $datasetProperties->{'name'};
+		my $origin = $datasetProperties->{'origin'};
+		if ($origin eq "-") {
+			$origin = undef;
+		} else {
+			# check if clone source is replicated too
+			my @values = split(/@/, $origin, 2);
+			my $srcdataset = $values[0];
+
+			my $found = 0;
+			foreach my $datasetProperties(@datasets) {
+				if ($datasetProperties->{'name'} eq $srcdataset) {
+					$found = 1;
+					last;
+				}
+			}
+
+			if ($found == 0) {
+				# clone source is not replicated, do a full replication
+				$origin = undef;
+			} else {
+				# clone source is replicated, defer until all non clones are replicated
+				push @deferred, $datasetProperties;
+				next;
+			}
+		}
+
 		$dataset =~ s/\Q$sourcefs\E//;
 		chomp $dataset;
 		my $childsourcefs = $sourcefs . $dataset;
 		my $childtargetfs = $targetfs . $dataset;
 		# print "syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs); \n";
-		syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs);
+		syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin);
+	}
+
+	# replicate cloned datasets and if this is the initial run, recreate them on the target
+	foreach my $datasetProperties(@deferred) {
+		my $dataset = $datasetProperties->{'name'};
+		my $origin = $datasetProperties->{'origin'};
+
+		$dataset =~ s/\Q$sourcefs\E//;
+		chomp $dataset;
+		my $childsourcefs = $sourcefs . $dataset;
+		my $childtargetfs = $targetfs . $dataset;
+		syncdataset($sourcehost, $childsourcefs, $targethost, $childtargetfs, $origin);
 	}
 }
 
@@ -147,37 +189,51 @@ sub getchilddatasets {
 		$fsescaped = escapeshellparam($fsescaped);
 	}
 
-	my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fsescaped |";
+	my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name,origin -t filesystem,volume -Hr $fsescaped |";
 	if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; }
-	open FH, $getchildrencmd;
-	my @children = <FH>;
-	close FH;
-
-	if (defined $args{'skip-parent'}) {
-		# parent dataset is the first element
-		shift @children;
+	if (! open FH, $getchildrencmd) {
+		die "ERROR: list command failed!\n";
 	}
 
-	if (defined $args{'exclude'}) {
-		my $excludes = $args{'exclude'};
-		foreach (@$excludes) {
-			for my $i ( 0 .. $#children ) {
-				if ($children[$i] =~ /$_/) {
-					if ($debug) { print "DEBUG: excluded $children[$i] because of $_\n"; }
-					undef $children[$i]
+	my @children;
+	my $first = 1;
+
+	DATASETS: while(<FH>) {
+		chomp;
+
+		if (defined $args{'skip-parent'} && $first eq 1) {
+			# parent dataset is the first element
+			$first = 0;
+			next;
+		}
+
+		my ($dataset, $origin) = /^([^\t]+)\t([^\t]+)/;
+
+		if (defined $args{'exclude'}) {
+			my $excludes = $args{'exclude'};
+			foreach (@$excludes) {
+				print("$dataset\n");
+				if ($dataset =~ /$_/) {
+					if ($debug) { print "DEBUG: excluded $dataset because of $_\n"; }
+					next DATASETS;
 				}
 			}
-
-			@children = grep{ defined }@children;
 		}
+
+		my %properties;
+		$properties{'name'} = $dataset;
+		$properties{'origin'} = $origin;
+
+		push @children, \%properties;
 	}
+	close FH;
 
 	return @children;
 }
 
 sub syncdataset {
 
-	my ($sourcehost, $sourcefs, $targethost, $targetfs) = @_;
+	my ($sourcehost, $sourcefs, $targethost, $targetfs, $origin) = @_;
 
 	my $sourcefsescaped = escapeshellparam($sourcefs);
 	my $targetfsescaped = escapeshellparam($targetfs);
@@ -305,11 +361,25 @@ sub syncdataset {
 		my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped";
 		my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
 
-		my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot);
+		my $pvsize;
+		if (defined $origin) {
+			my $originescaped = escapeshellparam($origin);
+			$sendcmd = "$sourcesudocmd $zfscmd send -i $originescaped $sourcefsescaped\@$oldestsnapescaped";
+			my $streamargBackup = $args{'streamarg'};
+			$args{'streamarg'} = "-i";
+			$pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$oldestsnap",$sourceisroot);
+			$args{'streamarg'} = $streamargBackup;
+		} else {
+			$pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot);
+		}
+
 		my $disp_pvsize = readablebytes($pvsize);
 		if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; }
 		my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
 		if (!$quiet) {
+			if (defined $origin) {
+				print "INFO: Clone is recreated on target $targetfs based on $origin\n";
+			}
 			if (!defined ($args{'no-stream'}) ) {
 				print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n";
 			} else {
@@ -396,7 +466,7 @@ sub syncdataset {
 
 		    # a resumed transfer will only be done to the next snapshot,
 		    # so do an normal sync cycle
-		    return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs);
+		    return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef);
 		}
 
 		# find most recent matching snapshot and do an -I

From 9d6cb42f4d2927c34d9d6344fbadb57f9f72caa1 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 21 Nov 2018 18:08:38 +0100
Subject: [PATCH 54/68] added option to disable smart clone handling

---
 README.md | 5 +++++
 syncoid   | 6 ++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b833dec..ed7a107 100644
--- a/README.md
+++ b/README.md
@@ -198,6 +198,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to not use resumeable zfs send/receive streams.
 
++ --no-clone-handling
+
+	This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead.
+
+
 + --dumpsnaps
 
 	This prints a list of snapshots during the run.
diff --git a/syncoid b/syncoid
index e9e3f27..c54c915 100755
--- a/syncoid
+++ b/syncoid
@@ -19,7 +19,8 @@ use Sys::Hostname;
 my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
-                   "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s") or pod2usage(2);
+                   "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
+                   "no-clone-handling") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 
@@ -114,7 +115,7 @@ if (!defined $args{'recursive'}) {
 	foreach my $datasetProperties(@datasets) {
 		my $dataset = $datasetProperties->{'name'};
 		my $origin = $datasetProperties->{'origin'};
-		if ($origin eq "-") {
+		if ($origin eq "-" || defined $args{'no-clone-handling'}) {
 			$origin = undef;
 		} else {
 			# check if clone source is replicated too
@@ -1320,3 +1321,4 @@ Options:
   --dumpsnaps           Dumps a list of snapshots during the run
   --no-command-checks   Do not check command existence before attempting transfer. Not recommended
   --no-resume           Don't use the ZFS resume feature if available
+  --no-clone-handling   Don't try to recreate clones on target

From f153810d08f86e804f7e43febb812ddabe810e92 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 21 Nov 2018 23:14:30 +0100
Subject: [PATCH 55/68] check for valid estimated send size to prevent a perl
 warning on systems which doesn't output size informations

---
 syncoid | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/syncoid b/syncoid
index 30aef49..1f759e2 100755
--- a/syncoid
+++ b/syncoid
@@ -1154,6 +1154,11 @@ sub getsendsize {
 	}
 	chomp $sendsize;
 
+	# check for valid value
+	if ($sendsize !~ /^\d+$/) {
+		$sendsize = '';
+	}
+
 	# to avoid confusion with a zero size pv, give sendsize
 	# a minimum 4K value - or if empty, make sure it reads UNKNOWN
 	if ($debug) { print "DEBUG: sendsize = $sendsize\n"; }

From a25ec83812ace12c26ba01296e278a2799d04d77 Mon Sep 17 00:00:00 2001
From: Rodger Donaldson <rodgerd@diaspora.gen.nz>
Date: Sun, 25 Nov 2018 07:14:59 +1300
Subject: [PATCH 56/68] Add a dependency for the configini patch

The RPM will install but fail to run if the perl-Config-IniFiles rpm is not also installed; this adds as a Requires: so that yum/dnf can find and install.
---
 packages/rhel/sanoid.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec
index 3a9412f..7d4995d 100644
--- a/packages/rhel/sanoid.spec
+++ b/packages/rhel/sanoid.spec
@@ -14,7 +14,7 @@ License:	   GPLv3
 URL:		   https://github.com/jimsalterjrs/sanoid
 Source0:           https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz
 
-Requires:	   perl, mbuffer, lzop, pv
+Requires:	   perl, mbuffer, lzop, pv, perl-Config-IniFiles
 %if 0%{?_with_systemd}
 Requires:      systemd >= 212
 

From ea55308dfcdce7614306d8a4b7187ff61d43c4b8 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Dec 2018 15:51:51 +0100
Subject: [PATCH 57/68] implemented support for excluding children of a
 specific dataset

---
 sanoid               | 18 ++++++++++++++----
 sanoid.defaults.conf |  1 +
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/sanoid b/sanoid
index 7ae1b5b..8f390ba 100755
--- a/sanoid
+++ b/sanoid
@@ -661,7 +661,7 @@ sub init {
 	tie my %ini, 'Config::IniFiles', ( -file => $conf_file ) or die "FATAL: cannot load $conf_file - please create a valid local config file before running sanoid!";
 
 	# we'll use these later to normalize potentially true and false values on any toggle keys
-	my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only');
+	my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive','process_children_only','skip_children');
 	my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
 	my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
 
@@ -718,7 +718,7 @@ sub init {
 
 		# override with any locally set values in the module itself
 		foreach my $key (keys %{$ini{$section}} ) {
-			if (! ($key =~ /template|recursive/)) {
+			if (! ($key =~ /template|recursive|skip_children/)) {
 				if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value directly set in module.\n"; }
 				$config{$section}{$key} = $ini{$section}{$key};
 			}
@@ -743,10 +743,17 @@ sub init {
 
 		# how 'bout some recursion? =)
 		my @datasets;
-		if ($ini{$section}{'recursive'}) {
+		if ($ini{$section}{'recursive'} || $ini{$section}{'skip_children'}) {
 			@datasets = getchilddatasets($config{$section}{'path'});
-			foreach my $dataset(@datasets) {
+			DATASETS: foreach my $dataset(@datasets) {
 				chomp $dataset;
+
+				if ($ini{$section}{'skip_children'}) {
+					if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; }
+					delete $config{$dataset};
+					next DATASETS;
+				}
+
 				foreach my $key (keys %{$config{$section}} ) {
 					if (! ($key =~ /template|recursive|children_only/)) {
 						if ($args{'debug'}) { print "DEBUG: recursively setting $key from $section to $dataset.\n"; }
@@ -1257,6 +1264,9 @@ sub getchilddatasets {
 	my @children = <FH>;
 	close FH;
 
+	# parent dataset is the first element
+	shift @children;
+
 	return @children;
 }
 
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index d86cc47..0c9037a 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -15,6 +15,7 @@ path =
 recursive =
 use_template =
 process_children_only =
+skip_children =
 
 # If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
 # prune any of those type snapshots already present.

From a0b983ee6ef791449fb631353e26b4f05a5ab71a Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Dec 2018 21:38:15 +0100
Subject: [PATCH 58/68] warn if unknown interval type is used

---
 sanoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanoid b/sanoid
index cb43066..5c81893 100755
--- a/sanoid
+++ b/sanoid
@@ -471,7 +471,7 @@ sub take_snapshots {
 					$lastpreferred = timelocal(@preferredtime);
 					if ($lastpreferred > time()) { $lastpreferred -= 60*60*24*31*365.25; } # preferred time is later this year - so look at last year
 				} else {
-					# unknown type
+					warn "WARN: unknown interval type $type in config!";
 					next;
 				}
 

From 7d742914e53591cfb2d13ad9dc22df55e9350446 Mon Sep 17 00:00:00 2001
From: Ben Wolsieffer <benwolsieffer@gmail.com>
Date: Wed, 23 Aug 2017 19:23:22 -0400
Subject: [PATCH 59/68] Add '--no-privilege-elevation' option to bypass root
 check.

---
 README.md | 4 ++++
 syncoid   | 7 ++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index c537540..e86d13a 100644
--- a/README.md
+++ b/README.md
@@ -215,6 +215,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This prints a list of snapshots during the run.
 
++ --no-privilege-elevation
+
+	Bypass the root check and assume syncoid has the necessary permissions (for use with ZFS permission delegation).
+
 + --sshport
 
 	Allow sync to/from boxes running SSH on non-standard ports.
diff --git a/syncoid b/syncoid
index e960d53..ccc5861 100755
--- a/syncoid
+++ b/syncoid
@@ -20,7 +20,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
-                   "no-clone-handling") or pod2usage(2);
+                   "no-clone-handling", "no-privilege-elevation") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 
@@ -1186,7 +1186,7 @@ sub getssh {
 		$rhost =~ s/:\Q$fs\E$//;
 		my $remoteuser = $rhost;
 		 $remoteuser =~ s/\@.*$//;
-		if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; }
+		if ($remoteuser eq 'root' || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; }
 		# now we need to establish a persistent master SSH connection
 		$socket = "/tmp/syncoid-$remoteuser-$rhost-" . time();
 		open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $args{'sshport'} $rhost exit |";
@@ -1194,7 +1194,7 @@ sub getssh {
 		$rhost = "-S $socket $rhost";
 	} else {
 		my $localuid = $<;
-		if ($localuid == 0) { $isroot = 1; } else { $isroot = 0; }
+		if ($localuid == 0 || $args{'no-privilege-elevation'}) { $isroot = 1; } else { $isroot = 0; }
 	}
 	# if ($isroot) { print "this user is root.\n"; } else { print "this user is not root.\n"; }
 	return ($rhost,$fs,$isroot);
@@ -1455,3 +1455,4 @@ Options:
   --no-command-checks   Do not check command existence before attempting transfer. Not recommended
   --no-resume           Don't use the ZFS resume feature if available
   --no-clone-handling   Don't try to recreate clones on target
+  --no-privilege-elevation  Bypass the root check, for use with ZFS permission delegation

From 23a0ce2e06a8bf091335aa8fb6ffb3356bc549a2 Mon Sep 17 00:00:00 2001
From: Jim Salter <github@jrs-s.net>
Date: Tue, 4 Dec 2018 16:33:03 -0500
Subject: [PATCH 60/68] Update CHANGELIST

---
 CHANGELIST | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/CHANGELIST b/CHANGELIST
index 515d05d..5c53199 100644
--- a/CHANGELIST
+++ b/CHANGELIST
@@ -1,3 +1,29 @@
+current PRERELEASE:
+       #140 - --no-privilege-elevation option to bypass root checks entirely
+       #157 - configurable frequent snapshot period
+       #163 - weekly snapshot period
+       #247 - implement support for zfs bookmarks in syncoid
+       #248 - allow forced target snapshot deletion in syncoid with --force-delete, for those who like to live dangerously
+       #254 - add xz compression option to syncoid
+       #261 - add WARNings for unsupported parameters in templates
+       #262 - documentation typo fix
+       #264 - add clean WARN error for unknown snapshot type values
+       #265 - add support for lz4 compression to syncoid
+       #266 - squash uninitialized value perl warning on first sanoid run (no snapshots)
+       #267 - improve boolean handling for recursive parameter
+       #268 - update README
+       #269 - don't alarm for lack of hourly/daily/etc if hourly/daily/etc=0
+       #277 - update automated test script for DST 
+       #279 - FreeBSD pkg typo fix
+       #280 - add pre,post,prune snapshot script hooks
+       #286 - remove 1s sleep interval between snapshots
+       #287 - run snapshot prune, snapshot take as separate systemd services in Debian package
+       #289 - optional defer pruning until low %FREE reached
+       #290 - replicate clone structure from source to target
+       #292 - fix sendsize estimation to prevent perl warnings on some systems
+       #294 - add dependency for Perl::Config::Ini
+       #298 - exclude recursion for children of a dataset with skip-children = yes
+
 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid,
        thank you @phreaker0!
 

From 00a9c6f6b551ef1d19c97ecba102f9a47388fd5d Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Dec 2018 23:12:16 +0100
Subject: [PATCH 61/68] added changed from recently merged PR's

---
 CHANGELIST                | 60 +++++++++++++++++++++++--------------
 packages/debian/changelog | 62 ++++++++++++++++++++++++---------------
 packages/rhel/sanoid.spec |  2 +-
 3 files changed, 78 insertions(+), 46 deletions(-)

diff --git a/CHANGELIST b/CHANGELIST
index 3ef74ae..9172b2d 100644
--- a/CHANGELIST
+++ b/CHANGELIST
@@ -1,25 +1,41 @@
-1.4.19 [sanoid]  monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
-       [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
-       [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
-       [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0)
-       [syncoid] make local source bwlimit work (@phreaker0)
-       [syncoid] fix 'resume support' detection on FreeBSD (@pit3k)
-       [sanoid]  updated INSTALL with missing dependency
-       [sanoid]  fixed monitor-health command for pools containing cache and log devices (@phreaker0)
-       [sanoid]  quiet flag suppresses all info output (@martinvw)
-       [sanoid]  check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
-       [sanoid]  added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
-       [sanoid]  cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
-       [sanoid]  implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
-       [syncoid] Added support for ZStandard compression.(@danielewood)
-       [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0)
-       [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
-       [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
-       [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0)
-       [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0)
-       [sanoid]  use UTC by default in unit template and documentation (@phreaker0)
-       [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0)
-       [syncoid] documented compatibility issues with (t)csh shells (@ecoutu)
+1.4.19 [overall]   documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0)
+       [syncoid]   added force delete flag (@phreaker0)
+       [sanoid]	   removed sleeping between snapshot taking (@phreaker0)
+       [syncoid]   added '--no-privilege-elevation' option to bypass root check (@lopsided98)
+       [sanoid]	   implemented weekly period (@phreaker0)
+       [syncoid]   implemented support for zfs bookmarks as fallback (@phreaker0)
+       [sanoid]    support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0)
+       [sanoid]    ignore snapshots types that are set to 0 (@muff1nman)
+       [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0)
+       [syncoid]   replicate clones (@phreaker0)
+       [syncoid]   added compression algorithms: lz4, xz (@spheenik, @phreaker0)
+       [sanoid]    added option to defer pruning based on the available pool capacity (@phreaker0)
+       [sanoid]    implemented frequent snapshots with configurable period (@phreaker0)
+       [syncoid]   prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0)
+       [packaging] dependency fixes (@rodgerd, mabushey)
+       [syncoid]   implemented support for excluding children of a specific dataset (@phreaker0)
+       [sanoid]    monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
+       [syncoid]   added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
+       [syncoid]   don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
+       [syncoid]   return a non zero exit code if there was a problem replicating datasets (@phreaker0)
+       [syncoid]   make local source bwlimit work (@phreaker0)
+       [syncoid]   fix 'resume support' detection on FreeBSD (@pit3k)
+       [sanoid]    updated INSTALL with missing dependency
+       [sanoid]    fixed monitor-health command for pools containing cache and log devices (@phreaker0)
+       [sanoid]    quiet flag suppresses all info output (@martinvw)
+       [sanoid]    check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
+       [sanoid]    added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
+       [sanoid]    cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
+       [sanoid]    implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
+       [syncoid]   Added support for ZStandard compression.(@danielewood)
+       [syncoid]   implemented support for excluding datasets from replication with regular expressions (@phreaker0)
+       [syncoid]   correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
+       [syncoid]   added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
+       [syncoid]   added option for skipping the parent dataset in recursive replication (@phreaker0)
+       [syncoid]   typos (@UnlawfulMonad, @jsavikko, @phreaker0)
+       [sanoid]    use UTC by default in unit template and documentation (@phreaker0)
+       [syncoid]   don't prune snapshots if instructed to not create them either (@phreaker0)
+       [syncoid]   documented compatibility issues with (t)csh shells (@ecoutu)
 
 1.4.18 implemented special character handling and support of ZFS resume/receive tokens by default in syncoid,
        thank you @phreaker0!
diff --git a/packages/debian/changelog b/packages/debian/changelog
index 67cca4d..34909b6 100644
--- a/packages/debian/changelog
+++ b/packages/debian/changelog
@@ -1,29 +1,45 @@
 sanoid (1.4.19) unstable; urgency=medium
 
-  [sanoid]  monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
-  [syncoid] added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
-  [syncoid] don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
-  [syncoid] return a non zero exit code if there was a problem replicating datasets (@phreaker0)
-  [syncoid] make local source bwlimit work (@phreaker0)
-  [syncoid] fix 'resume support' detection on FreeBSD (@pit3k)
-  [sanoid]  updated INSTALL with missing dependency
-  [sanoid]  fixed monitor-health command for pools containing cache and log devices (@phreaker0)
-  [sanoid]  quiet flag suppresses all info output (@martinvw)
-  [sanoid]  check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
-  [sanoid]  added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
-  [sanoid]  cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
-  [sanoid]  implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
-  [syncoid] Added support for ZStandard compression.(@danielewood)
-  [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0)
-  [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
-  [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
-  [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0)
-  [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0)
-  [sanoid]  use UTC by default in unit template and documentation (@phreaker0)
-  [syncoid] don't prune snapshots if instructed to not create them either (@phreaker0)
-  [syncoid] documented compatibility issues with (t)csh shells (@ecoutu)
+  [overall]   documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0)
+  [syncoid]   added force delete flag (@phreaker0)
+  [sanoid]	  removed sleeping between snapshot taking (@phreaker0)
+  [syncoid]   added '--no-privilege-elevation' option to bypass root check (@lopsided98)
+  [sanoid]	  implemented weekly period (@phreaker0)
+  [syncoid]   implemented support for zfs bookmarks as fallback (@phreaker0)
+  [sanoid]    support for pre, post and prune snapshot scripts (@jouir, @darkbasic, @phreaker0)
+  [sanoid]    ignore snapshots types that are set to 0 (@muff1nman)
+  [packaging] split snapshot taking/pruning into separate systemd units for debian package (@phreaker0)
+  [syncoid]   replicate clones (@phreaker0)
+  [syncoid]   added compression algorithms: lz4, xz (@spheenik, @phreaker0)
+  [sanoid]    added option to defer pruning based on the available pool capacity (@phreaker0)
+  [sanoid]    implemented frequent snapshots with configurable period (@phreaker0)
+  [syncoid]   prevent a perl warning on systems which doesn't output estimated send size information (@phreaker0)
+  [packaging] dependency fixes (@rodgerd, mabushey)
+  [syncoid]   implemented support for excluding children of a specific dataset (@phreaker0)
+  [sanoid]    monitor-health command additionally checks vdev members for io and checksum errors (@phreaker0)
+  [syncoid]   added ability to skip datasets by a custom dataset property 'syncoid:no-sync' (@attie)
+  [syncoid]   don't die on some critical replication errors, but continue with the remaining datasets (@phreaker0)
+  [syncoid]   return a non zero exit code if there was a problem replicating datasets (@phreaker0)
+  [syncoid]   make local source bwlimit work (@phreaker0)
+  [syncoid]   fix 'resume support' detection on FreeBSD (@pit3k)
+  [sanoid]    updated INSTALL with missing dependency
+  [sanoid]    fixed monitor-health command for pools containing cache and log devices (@phreaker0)
+  [sanoid]    quiet flag suppresses all info output (@martinvw)
+  [sanoid]    check for empty lockfile which lead to sanoid failing on start (@jasonblewis)
+  [sanoid]    added dst handling to prevent multiple invalid snapshots on time shift (@phreaker0)
+  [sanoid]    cache improvements, makes sanoid much faster with a huge amount of datasets/snapshots (@phreaker0)
+  [sanoid]    implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0)
+  [syncoid]   Added support for ZStandard compression.(@danielewood)
+  [syncoid]   implemented support for excluding datasets from replication with regular expressions (@phreaker0)
+  [syncoid]   correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0)
+  [syncoid]   added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0)
+  [syncoid]   added option for skipping the parent dataset in recursive replication (@phreaker0)
+  [syncoid]   typos (@UnlawfulMonad, @jsavikko, @phreaker0)
+  [sanoid]    use UTC by default in unit template and documentation (@phreaker0)
+  [syncoid]   don't prune snapshots if instructed to not create them either (@phreaker0)
+  [syncoid]   documented compatibility issues with (t)csh shells (@ecoutu)
 
- -- Jim Salter <github@jrs-s.net>  Wed, 05 Sep 2018 04:00:00 -0400
+ -- Jim Salter <github@jrs-s.net>  Wed, 04 Dec 2018 18:10:00 -0400
 
 sanoid (1.4.18) unstable; urgency=medium
 
diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec
index b5d7172..1579a27 100644
--- a/packages/rhel/sanoid.spec
+++ b/packages/rhel/sanoid.spec
@@ -111,7 +111,7 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name}
 %endif
 
 %changelog
-* Wed Sep 05 2018 Christoph Klaffl <christoph@phreaker.eu> - 1.4.19
+* Wed Dec 04 2018 Christoph Klaffl <christoph@phreaker.eu> - 1.4.19
 - Bump to 1.4.19
 * Sat Apr 28 2018 Dominic Robinson <github@dcrdev.com> - 1.4.18-1
 - Bump to 1.4.18

From 3a1ffe8554802dfc2459032b3beba5f5cf360640 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 4 Dec 2018 23:33:18 +0100
Subject: [PATCH 62/68] fixed a regression which causes perl warnings

---
 sanoid | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sanoid b/sanoid
index fbb94b6..dc71b9d 100755
--- a/sanoid
+++ b/sanoid
@@ -843,13 +843,15 @@ sub init {
 		}
 
 		# how 'bout some recursion? =)
+		my $recursive = $ini{$section}{'recursive'} && grep( /^$ini{$section}{'recursive'}$/, @istrue );
+		my $skipChildren = $ini{$section}{'skip_children'} && grep( /^$ini{$section}{'skip_children'}$/, @istrue );
 		my @datasets;
-		if (grep( /^$ini{$section}{'recursive'}$/, @istrue ) || grep( /^$ini{$section}{'skip_children'}$/, @istrue )) {
+		if ($recursive || $skipChildren) {
 			@datasets = getchilddatasets($config{$section}{'path'});
 			DATASETS: foreach my $dataset(@datasets) {
 				chomp $dataset;
 
-				if (grep( /^$ini{$section}{'skip_children'}$/, @istrue )) {
+				if ($skipChildren) {
 					if ($args{'debug'}) { print "DEBUG: ignoring $dataset.\n"; }
 					delete $config{$dataset};
 					next DATASETS;

From 1ed37e9891400755311429a4a21e5102c87cd393 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Thu, 6 Dec 2018 09:26:22 +0100
Subject: [PATCH 63/68] Resolve a conflict

---
 syncoid | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/syncoid b/syncoid
index 3ad1622..1bb4c80 100755
--- a/syncoid
+++ b/syncoid
@@ -583,14 +583,6 @@ sub syncdataset {
 					system ("$targetsudocmd $zfscmd rollback $rollbacktype $targetfsescaped\@$matchingsnapescaped");
 				}
 			}
-<<<<<<< HEAD
-			my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
-			my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
-			my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
-			my $disp_pvsize = readablebytes($pvsize);
-			if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
-			my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
-=======
 
 			my $nextsnapshot = 0;
 
@@ -608,7 +600,6 @@ sub syncdataset {
 						}
 					}
 				}
->>>>>>> e186f3c66e9c757fa62c4eaa8a1c05bc49dbcff1
 
 				# bookmark stream size can't be determined
 				my $pvsize = 0;

From 210e0aae640481cc846609a9947c66d7a5b2b82f Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Fri, 7 Dec 2018 17:38:45 +0100
Subject: [PATCH 64/68] Introduced (un)forced recv when using bookmarks also

---
 syncoid | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/syncoid b/syncoid
index 1bb4c80..f599f96 100755
--- a/syncoid
+++ b/syncoid
@@ -609,7 +609,7 @@ sub syncdataset {
 					my $nextsnapshotescaped = escapeshellparam($nextsnapshot);
 
 					my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped";
-					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 					my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
 
 					if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; }
@@ -624,7 +624,7 @@ sub syncdataset {
 					$matchingsnapescaped = escapeshellparam($matchingsnap);
 				} else {
 					my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped";
-					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+					my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 					my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
 
 					if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; }
@@ -641,7 +641,7 @@ sub syncdataset {
 			# bookmark replication was only done to the next oldest snapshot
 			if (!$bookmark || $nextsnapshot) {
 				my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped";
-				my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
+				my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs $forcedrecv $targetfsescaped";
 				my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
 				my $disp_pvsize = readablebytes($pvsize);
 				if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }

From 8568ac3e1030751ce5726bd4950c3e7e20f4b8f2 Mon Sep 17 00:00:00 2001
From: Gionatan Danti <g.danti@assyoma.it>
Date: Mon, 10 Dec 2018 11:21:43 +0100
Subject: [PATCH 65/68] Fix missing coma

---
 syncoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncoid b/syncoid
index f599f96..d3e678f 100755
--- a/syncoid
+++ b/syncoid
@@ -20,7 +20,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [
 GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
-                   "no-clone-handling", "no-privilege-elevation", "force-delete" "no-clone-rollback", "no-rollback") or pod2usage(2);
+                   "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback") or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
 

From dc4df15e2ee8988449788861b7bd8f0260ea94f3 Mon Sep 17 00:00:00 2001
From: Jim Salter <github@jrs-s.net>
Date: Fri, 14 Dec 2018 10:44:45 -0500
Subject: [PATCH 66/68] fix broken monthly_warn monthly_crit in
 sanoid.defaults.conf

---
 sanoid.defaults.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 4139393..96be95c 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -100,8 +100,8 @@ daily_warn = 28
 daily_crit = 32
 weekly_warn = 0
 weekly_crit = 0
-monthly_warn = 5
-monthly_crit = 6
+monthly_warn = 32
+monthly_crit = 40
 yearly_warn = 0
 yearly_crit = 0
 

From 2ece13eccf0bd4f5ce0aaf15c2bfdffed37b9d41 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Sun, 16 Dec 2018 21:54:43 +0100
Subject: [PATCH 67/68] allow time units to be used for monitoring warn/crit
 values

---
 sanoid               | 48 ++++++++++++++++++++++++++++++++++++++++----
 sanoid.defaults.conf | 12 +++++------
 2 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/sanoid b/sanoid
index ddd457f..3aa57a3 100755
--- a/sanoid
+++ b/sanoid
@@ -143,8 +143,8 @@ sub monitor_snapshots {
 
 			my $typewarn = $type . '_warn';
 			my $typecrit = $type . '_crit';
-			my $warn = $config{$section}{$typewarn} * $smallerperiod;
-			my $crit = $config{$section}{$typecrit} * $smallerperiod;
+			my $warn = convertTimePeriod($config{$section}{$typewarn}, $smallerperiod);
+			my $crit = convertTimePeriod($config{$section}{$typecrit}, $smallerperiod);
 			my $elapsed = -1;
 			if (defined $snapsbytype{$path}{$type}{'newest'}) {
 				$elapsed = $snapsbytype{$path}{$type}{'newest'};
@@ -153,7 +153,7 @@ sub monitor_snapshots {
 			my $dispwarn = displaytime($warn);
 			my $dispcrit = displaytime($crit);
 			if ( $elapsed > $crit || $elapsed == -1) {
-				if ($config{$section}{$typecrit} > 0) {
+				if ($crit > 0) {
 					if (! $config{$section}{'monitor_dont_crit'}) { $errorlevel = 2; }
 					if ($elapsed == -1) {
 						push @msgs, "CRIT: $path has no $type snapshots at all!";
@@ -162,7 +162,7 @@ sub monitor_snapshots {
 					}
 				}
 			 } elsif ($elapsed > $warn) {
-				if ($config{$section}{$typewarn} > 0) {
+				if ($warn > 0) {
 					if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; }
 					push @msgs, "WARN: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispwarn)";
 				}
@@ -1511,6 +1511,46 @@ sub runscript {
 	return $ret;
 }
 
+#######################################################################################################################3
+#######################################################################################################################3
+#######################################################################################################################3
+
+sub convertTimePeriod {
+	my $value=shift;
+	my $period=shift;
+
+	if ($value =~ /^\d+Y$/) {
+		$period = 60*60*24*31*365;
+		chop $value;
+	} elsif ($value =~ /^\d+M$/) {
+		$period = 60*60*24*31;
+		chop $value;
+	} elsif ($value =~ /^\d+W$/) {
+		$period = 60*60*24*7;
+		chop $value;
+	} elsif ($value =~ /^\d+D$/) {
+		$period = 60*60*24;
+		chop $value;
+	} elsif ($value =~ /^\d+h$/) {
+		$period = 60*60;
+		chop $value;
+	} elsif ($value =~ /^\d+m$/) {
+		$period = 60;
+		chop $value;
+	} elsif ($value =~ /^\d+s$/) {
+		$period = 1;
+		chop $value;
+	} elsif ($value =~ /^\d+$/) {
+		# no unit, provided fallback period is used
+	} else {
+		# invalid value, return smallest valid value as fallback
+		# (will trigger a warning message for monitoring for sure)
+		return 1;
+	}
+
+	return $value * $period;
+}
+
 __END__
 
 =head1 NAME
diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 96be95c..8785e7c 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -94,14 +94,14 @@ monitor_dont_warn = no
 monitor_dont_crit = no
 frequently_warn = 0
 frequently_crit = 0
-hourly_warn = 90
-hourly_crit = 360
-daily_warn = 28
-daily_crit = 32
+hourly_warn = 90m
+hourly_crit = 360m
+daily_warn = 28h
+daily_crit = 32h
 weekly_warn = 0
 weekly_crit = 0
-monthly_warn = 32
-monthly_crit = 40
+monthly_warn = 32D
+monthly_crit = 40D
 yearly_warn = 0
 yearly_crit = 0
 

From cfab4eafdf2e8512f787db9e854835872493533f Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Sun, 16 Dec 2018 22:02:14 +0100
Subject: [PATCH 68/68] added/fixed documentation

---
 sanoid.defaults.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf
index 8785e7c..6649c2e 100644
--- a/sanoid.defaults.conf
+++ b/sanoid.defaults.conf
@@ -83,7 +83,8 @@ yearly_min = 0
 # monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down
 # example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old,
 # daily_crit = 36 means issue CRITICAL if most recent daily snapshot is not less than 36 hours old,
-# monthly_warn = 36 means issue WARNING if most recent monthly snapshot is not less than 36 days old... etc.
+# monthly_warn = 5 means issue WARNING if most recent monthly snapshot is not less than 5 weeks old... etc.
+# the following time suffixes can also be used: Y = years, M = months, W = weeks, D = days, h = hours, m = minutes, s = seconds
 #
 # monitor_dont_warn = yes will cause the monitoring service to report warnings as text, but with status OK.
 # monitor_dont_crit = yes will cause the monitoring service to report criticals as text, but with status OK.