From 84bc3164f0414da416b13e9902dc8680631add9d Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 21 Feb 2020 20:34:28 +0100
Subject: [PATCH 01/22] implemented option for direct connection via socat and
 busybox nc

---
 README.md |  5 +++++
 syncoid   | 44 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c2b502b..2afe3b0 100644
--- a/README.md
+++ b/README.md
@@ -262,6 +262,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	Use specified identity file as per ssh -i.
 
++ --insecure-direct-connection=IP:PORT[,IP:PORT]
+
+	WARNING: This is an insecure option as the data is not encrypted while being sent over the network. Only use if you trust the complete network path.
+	Use a direct tcp connection (with socat and busybox nc) for the actual zfs send/recv stream. All control commands are still executed via the ssh connection. The first address pair is used for connecting to the target host from the source host and the second pair is for listening on the target host. If the later isn't provided the same as the former is used. This can be used for saturating high throughput connection like >= 10GBe network which isn't easy with the overhead off ssh. It can also be useful for encrypted datasets to lower the cpu usage needed for replication but be aware that metadata is NOT ENCRYPTED in this case. (This option can't be used for relaying between two remote hosts)
+
 + --quiet
 
 	Supress non-error output.
diff --git a/syncoid b/syncoid
index 2eef326..cef0e3a 100755
--- a/syncoid
+++ b/syncoid
@@ -25,7 +25,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback",
-                   "create-bookmark", "pv-options=s" => \$pvoptions,
+                   "create-bookmark", "pv-options=s" => \$pvoptions, "insecure-direct-connection=s",
                    "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -120,6 +120,31 @@ if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; }
 my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs);
 my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs);
 
+# handle insecure direct connection arguments
+my $directconnect = "";
+my $directlisten = "";
+
+if (length $args{'insecure-direct-connection'}) {
+	if ($sourcehost ne '' && $targethost ne '') {
+		print("CRITICAL: relaying between remote hosts is not supported with insecure direct connection!\n");
+		pod2usage(2);
+		exit 127;
+	}
+
+	my @parts = split(',', $args{'insecure-direct-connection'});
+	if (scalar @parts > 2) {
+		print("CRITICAL: invalid insecure-direct-connection argument!\n");
+		pod2usage(2);
+		exit 127;
+	} elsif (scalar @parts == 2) {
+		$directconnect = $parts[0];
+		$directlisten = $parts[1];
+	} else {
+		$directconnect = $args{'insecure-direct-connection'};
+		$directlisten = $args{'insecure-direct-connection'};
+	}
+}
+
 my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
 my $targetsudocmd = $targetisroot ? '' : $sudocmd;
 
@@ -1250,9 +1275,17 @@ sub buildsynccmd {
 		if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd $pvoptions -s $pvsize |"; }
 		if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; }
 		if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
+		if (length $directconnect) {
+			# try 10 times over 10 seconds to connect
+			$synccmd .= " socat - TCP:" . $directconnect . ",retry=10,interval=1 |";
+		}
 		$synccmd .= " $sshcmd $targethost ";
 
 		my $remotecmd = "";
+		if (length $directlisten) {
+			# wait up to 10 seconds for a connection or error out
+			$remotecmd .= " busybox nc -l " . $directlisten . " -w 10 |";
+		}
 		if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
 		if ($avail{'compress'}) { $remotecmd .= " $compressargs{'decomcmd'} |"; }
 		$remotecmd .= " $recvcmd";
@@ -1265,9 +1298,17 @@ sub buildsynccmd {
 		my $remotecmd = $sendcmd;
 		if ($avail{'compress'}) { $remotecmd .= " | $compressargs{'cmd'}"; }
 		if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
+		if (length $directconnect) {
+			# try 10 times over 10 seconds to connect
+			$remotecmd .= " | socat - TCP:" . $directconnect . ",retry=10,interval=1";
+		}
 
 		$synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd);
 		$synccmd .= " | ";
+		if (length $directlisten) {
+			# wait up to 10 seconds for a connection or error out
+			$synccmd .= " busybox nc -l " . $directlisten . " -w 10 |";
+		}
 		if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
 		if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; }
 		if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; }
@@ -1911,6 +1952,7 @@ Options:
   --sshport=PORT        Connects to remote on a particular port
   --sshcipher|c=CIPHER  Passes CIPHER to ssh to use a particular cipher set
   --sshoption|o=OPTION  Passes OPTION to ssh for remote usage. Can be specified multiple times
+  --insecure-direct-connection=IP:PORT[,IP:PORT]  WARNING: DATA IS NOT ENCRYPTED. First address pair is for connecting to the target and the second for listening at the target
 
   --help                Prints this helptext
   --version             Prints the version number

From 2e6abddb5e04b1acd18dba4c2be32b6838d0e49d Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 30 Mar 2020 19:01:54 +0200
Subject: [PATCH 02/22] fixed uninitialized value warning

---
 syncoid | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/syncoid b/syncoid
index cef0e3a..554f5dd 100755
--- a/syncoid
+++ b/syncoid
@@ -120,6 +120,14 @@ if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; }
 my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs);
 my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs);
 
+my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
+my $targetsudocmd = $targetisroot ? '' : $sudocmd;
+
+# figure out whether compression, mbuffering, pv
+# are available on source, target, local machines.
+# warn user of anything missing, then continue with sync.
+my %avail = checkcommands();
+
 # handle insecure direct connection arguments
 my $directconnect = "";
 my $directlisten = "";
@@ -145,14 +153,6 @@ if (length $args{'insecure-direct-connection'}) {
 	}
 }
 
-my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
-my $targetsudocmd = $targetisroot ? '' : $sudocmd;
-
-# figure out whether compression, mbuffering, pv
-# are available on source, target, local machines.
-# warn user of anything missing, then continue with sync.
-my %avail = checkcommands();
-
 my %snaps;
 my $exitcode = 0;
 

From f0dfd46c9f4410ef1923bb0147771152b061598a Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Mon, 30 Mar 2020 20:15:09 +0200
Subject: [PATCH 03/22] use mbuffer tcp listen socket instead of busybox nc

---
 syncoid | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/syncoid b/syncoid
index 554f5dd..ede92e0 100755
--- a/syncoid
+++ b/syncoid
@@ -1284,9 +1284,11 @@ sub buildsynccmd {
 		my $remotecmd = "";
 		if (length $directlisten) {
 			# wait up to 10 seconds for a connection or error out
-			$remotecmd .= " busybox nc -l " . $directlisten . " -w 10 |";
+			$remotecmd .= " $mbuffercmd $args{'target -bwlimit'} -W 10 -I " . $directlisten . " $mbufferoptions |";
+		} else {
+			if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
 		}
-		if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
+
 		if ($avail{'compress'}) { $remotecmd .= " $compressargs{'decomcmd'} |"; }
 		$remotecmd .= " $recvcmd";
 
@@ -1307,9 +1309,11 @@ sub buildsynccmd {
 		$synccmd .= " | ";
 		if (length $directlisten) {
 			# wait up to 10 seconds for a connection or error out
-			$synccmd .= " busybox nc -l " . $directlisten . " -w 10 |";
+			$synccmd .= "$mbuffercmd $args{'target-bwlimit'} -W 10 -I " . $directlisten . " $mbufferoptions | ";
+		} else {
+			if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
 		}
-		if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
+
 		if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; }
 		if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; }
 		$synccmd .= "$recvcmd";

From 807f6fa1bba3ab37ec30323a0bd6d4fcebee49b0 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 31 Mar 2020 09:27:17 +0200
Subject: [PATCH 04/22] check if socat and mbuffer is available for insecure
 direct connection

---
 syncoid | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/syncoid b/syncoid
index ede92e0..1f3de38 100755
--- a/syncoid
+++ b/syncoid
@@ -86,6 +86,7 @@ my $pscmd = 'ps';
 
 my $pvcmd = 'pv';
 my $mbuffercmd = 'mbuffer';
+my $socatcmd = 'socat';
 my $sudocmd = 'sudo';
 my $mbufferoptions = "-q -s 128k -m $mbuffer_size 2>/dev/null";
 # currently using POSIX compatible command to check for program existence because we aren't depending on perl
@@ -1032,6 +1033,14 @@ sub checkcommands {
 		$avail{'compress'} = 0;
 	}
 
+	if (length $args{'insecure-direct-connection'}) {
+		if ($debug) { print "DEBUG: checking availability of $socatcmd on source...\n"; }
+		my $socatAvailable = `$sourcessh $checkcmd $socatcmd 2>/dev/null`;
+		if ($socatAvailable eq '') {
+			die "CRIT: $socatcmd is needed on source for insecure direct connection!\n";
+		}
+	}
+
 	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; }
 	$avail{'sourcembuffer'} = `$sourcessh $checkcmd $mbuffercmd 2>/dev/null`;
 	if ($avail{'sourcembuffer'} eq '') {
@@ -1044,6 +1053,9 @@ sub checkcommands {
 	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; }
 	$avail{'targetmbuffer'} = `$targetssh $checkcmd $mbuffercmd 2>/dev/null`;
 	if ($avail{'targetmbuffer'} eq '') {
+		if (length $args{'insecure-direct-connection'}) {
+			die "CRIT: $mbuffercmd is needed on target for insecure direct connection!\n";
+		}
 		if (!$quiet) { print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; }
 		$avail{'targetmbuffer'} = 0;
 	} else {
@@ -1277,7 +1289,7 @@ sub buildsynccmd {
 		if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
 		if (length $directconnect) {
 			# try 10 times over 10 seconds to connect
-			$synccmd .= " socat - TCP:" . $directconnect . ",retry=10,interval=1 |";
+			$synccmd .= " $socatcmd - TCP:" . $directconnect . ",retry=10,interval=1 |";
 		}
 		$synccmd .= " $sshcmd $targethost ";
 
@@ -1302,7 +1314,7 @@ sub buildsynccmd {
 		if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
 		if (length $directconnect) {
 			# try 10 times over 10 seconds to connect
-			$remotecmd .= " | socat - TCP:" . $directconnect . ",retry=10,interval=1";
+			$remotecmd .= " | $socatcmd - TCP:" . $directconnect . ",retry=10,interval=1";
 		}
 
 		$synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd);

From fcae918ffafb87f67aa5fa15f13193cb8709e184 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 31 Mar 2020 09:40:36 +0200
Subject: [PATCH 05/22] updated readme regarding busybox nc -> mbuffer switch

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2afe3b0..7a3db82 100644
--- a/README.md
+++ b/README.md
@@ -265,7 +265,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 + --insecure-direct-connection=IP:PORT[,IP:PORT]
 
 	WARNING: This is an insecure option as the data is not encrypted while being sent over the network. Only use if you trust the complete network path.
-	Use a direct tcp connection (with socat and busybox nc) for the actual zfs send/recv stream. All control commands are still executed via the ssh connection. The first address pair is used for connecting to the target host from the source host and the second pair is for listening on the target host. If the later isn't provided the same as the former is used. This can be used for saturating high throughput connection like >= 10GBe network which isn't easy with the overhead off ssh. It can also be useful for encrypted datasets to lower the cpu usage needed for replication but be aware that metadata is NOT ENCRYPTED in this case. (This option can't be used for relaying between two remote hosts)
+	Use a direct tcp connection (with socat and mbuffer) for the actual zfs send/recv stream. All control commands are still executed via the ssh connection. The first address pair is used for connecting to the target host from the source host and the second pair is for listening on the target host. If the later isn't provided the same as the former is used. This can be used for saturating high throughput connection like >= 10GBe network which isn't easy with the overhead off ssh. It can also be useful for encrypted datasets to lower the cpu usage needed for replication but be aware that metadata is NOT ENCRYPTED in this case. (This option can't be used for relaying between two remote hosts)
 
 + --quiet
 

From e1a6507455f1e422a22a79be0c05543ba6205d26 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Thu, 2 Apr 2020 21:37:06 +0200
Subject: [PATCH 06/22] direct connection will default to busybox nc again but
 can be switched to mbuffer

---
 README.md |  4 ++--
 syncoid   | 66 +++++++++++++++++++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/README.md b/README.md
index 7a3db82..58c5a1a 100644
--- a/README.md
+++ b/README.md
@@ -262,10 +262,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	Use specified identity file as per ssh -i.
 
-+ --insecure-direct-connection=IP:PORT[,IP:PORT]
++ --insecure-direct-connection=IP:PORT[,IP:PORT,[TIMEOUT,[mbuffer]]]
 
 	WARNING: This is an insecure option as the data is not encrypted while being sent over the network. Only use if you trust the complete network path.
-	Use a direct tcp connection (with socat and mbuffer) for the actual zfs send/recv stream. All control commands are still executed via the ssh connection. The first address pair is used for connecting to the target host from the source host and the second pair is for listening on the target host. If the later isn't provided the same as the former is used. This can be used for saturating high throughput connection like >= 10GBe network which isn't easy with the overhead off ssh. It can also be useful for encrypted datasets to lower the cpu usage needed for replication but be aware that metadata is NOT ENCRYPTED in this case. (This option can't be used for relaying between two remote hosts)
+	Use a direct tcp connection (with socat and busybox nc/mbuffer) for the actual zfs send/recv stream. All control commands are still executed via the ssh connection. The first address pair is used for connecting to the target host from the source host and the second pair is for listening on the target host. If the later isn't provided the same as the former is used. This can be used for saturating high throughput connection like >= 10GBe network which isn't easy with the overhead off ssh. It can also be useful for encrypted datasets to lower the cpu usage needed for replication but be aware that metadata is NOT ENCRYPTED in this case. The default timeout is 60 seconds and can be overridden by providing it as third argument. By default busybox nc is used for the listeing tcp socket, if mbuffer is preferred specify its name as fourth argument but be aware that mbuffer listens on all interfaces and uses an optionally provided ip address for access restriction (This option can't be used for relaying between two remote hosts)
 
 + --quiet
 
diff --git a/syncoid b/syncoid
index 1f3de38..caaba44 100755
--- a/syncoid
+++ b/syncoid
@@ -124,14 +124,14 @@ my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs);
 my $sourcesudocmd = $sourceisroot ? '' : $sudocmd;
 my $targetsudocmd = $targetisroot ? '' : $sudocmd;
 
-# figure out whether compression, mbuffering, pv
-# are available on source, target, local machines.
-# warn user of anything missing, then continue with sync.
-my %avail = checkcommands();
+if (!defined $sourcehost) { $sourcehost = ''; }
+if (!defined $targethost) { $targethost = ''; }
 
 # handle insecure direct connection arguments
 my $directconnect = "";
 my $directlisten = "";
+my $directtimeout = 60;
+my $directmbuffer = 0;
 
 if (length $args{'insecure-direct-connection'}) {
 	if ($sourcehost ne '' && $targethost ne '') {
@@ -141,19 +141,34 @@ if (length $args{'insecure-direct-connection'}) {
 	}
 
 	my @parts = split(',', $args{'insecure-direct-connection'});
-	if (scalar @parts > 2) {
+	if (scalar @parts > 4) {
 		print("CRITICAL: invalid insecure-direct-connection argument!\n");
 		pod2usage(2);
 		exit 127;
-	} elsif (scalar @parts == 2) {
+	} elsif (scalar @parts >= 2) {
 		$directconnect = $parts[0];
 		$directlisten = $parts[1];
 	} else {
 		$directconnect = $args{'insecure-direct-connection'};
 		$directlisten = $args{'insecure-direct-connection'};
 	}
+
+	if (scalar @parts == 3) {
+		$directtimeout = $parts[2];
+	}
+
+	if (scalar @parts == 4) {
+		if ($parts[3] eq "mbuffer") {
+			$directmbuffer = 1;
+		}
+	}
 }
 
+# figure out whether compression, mbuffering, pv
+# are available on source, target, local machines.
+# warn user of anything missing, then continue with sync.
+my %avail = checkcommands();
+
 my %snaps;
 my $exitcode = 0;
 
@@ -965,9 +980,6 @@ sub checkcommands {
 		return %avail;
 	}
 
-	if (!defined $sourcehost) { $sourcehost = ''; }
-	if (!defined $targethost) { $targethost = ''; }
-
 	if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; }
 	if ($targethost ne '') { $targetssh = "$sshcmd $targethost"; } else { $targetssh = ''; }
 
@@ -1039,6 +1051,14 @@ sub checkcommands {
 		if ($socatAvailable eq '') {
 			die "CRIT: $socatcmd is needed on source for insecure direct connection!\n";
 		}
+
+		if (!$directmbuffer) {
+			if ($debug) { print "DEBUG: checking availability of busybox (for nc) on target...\n"; }
+			my $busyboxAvailable = `$targetssh $checkcmd busybox 2>/dev/null`;
+			if ($busyboxAvailable eq '') {
+				die "CRIT: busybox is needed on target for insecure direct connection!\n";
+			}
+		}
 	}
 
 	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; }
@@ -1053,7 +1073,7 @@ sub checkcommands {
 	if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; }
 	$avail{'targetmbuffer'} = `$targetssh $checkcmd $mbuffercmd 2>/dev/null`;
 	if ($avail{'targetmbuffer'} eq '') {
-		if (length $args{'insecure-direct-connection'}) {
+		if ($directmbuffer) {
 			die "CRIT: $mbuffercmd is needed on target for insecure direct connection!\n";
 		}
 		if (!$quiet) { print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; }
@@ -1288,19 +1308,18 @@ sub buildsynccmd {
 		if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; }
 		if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; }
 		if (length $directconnect) {
-			# try 10 times over 10 seconds to connect
-			$synccmd .= " $socatcmd - TCP:" . $directconnect . ",retry=10,interval=1 |";
+			$synccmd .= " $socatcmd - TCP:" . $directconnect . ",retry=$directtimeout,interval=1 |";
 		}
 		$synccmd .= " $sshcmd $targethost ";
 
 		my $remotecmd = "";
-		if (length $directlisten) {
-			# wait up to 10 seconds for a connection or error out
-			$remotecmd .= " $mbuffercmd $args{'target -bwlimit'} -W 10 -I " . $directlisten . " $mbufferoptions |";
-		} else {
-			if ($avail{'targetmbuffer'}) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
+		if ($directmbuffer) {
+			$remotecmd .= " $mbuffercmd $args{'target -bwlimit'} -W $directtimeout -I " . $directlisten . " $mbufferoptions |";
+		} elsif (length $directlisten) {
+			$remotecmd .= " busybox nc -l " . $directlisten . " -w $directtimeout |";
 		}
 
+		if ($avail{'targetmbuffer'} && !$directmbuffer) { $remotecmd .= " $mbuffercmd $args{'target-bwlimit'} $mbufferoptions |"; }
 		if ($avail{'compress'}) { $remotecmd .= " $compressargs{'decomcmd'} |"; }
 		$remotecmd .= " $recvcmd";
 
@@ -1313,19 +1332,18 @@ sub buildsynccmd {
 		if ($avail{'compress'}) { $remotecmd .= " | $compressargs{'cmd'}"; }
 		if ($avail{'sourcembuffer'}) { $remotecmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; }
 		if (length $directconnect) {
-			# try 10 times over 10 seconds to connect
-			$remotecmd .= " | $socatcmd - TCP:" . $directconnect . ",retry=10,interval=1";
+			$remotecmd .= " | $socatcmd - TCP:" . $directconnect . ",retry=$directtimeout,interval=1";
 		}
 
 		$synccmd = "$sshcmd $sourcehost " . escapeshellparam($remotecmd);
 		$synccmd .= " | ";
-		if (length $directlisten) {
-			# wait up to 10 seconds for a connection or error out
-			$synccmd .= "$mbuffercmd $args{'target-bwlimit'} -W 10 -I " . $directlisten . " $mbufferoptions | ";
-		} else {
-			if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
+		if ($directmbuffer) {
+			$synccmd .= "$mbuffercmd $args{'target-bwlimit'} -W $directtimeout -I " . $directlisten . " $mbufferoptions | ";
+		} elsif (length $directlisten) {
+			$synccmd .= " busybox nc -l " . $directlisten . " -w $directtimeout | ";
 		}
 
+		if ($avail{'targetmbuffer'} && !$directmbuffer) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; }
 		if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; }
 		if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; }
 		$synccmd .= "$recvcmd";

From 5092ff10001a3868d5a2e16a8e84728d5168efb9 Mon Sep 17 00:00:00 2001
From: Jim Perkins <jim@perkosolutions.com>
Date: Wed, 5 Aug 2020 11:37:48 -0500
Subject: [PATCH 07/22] typo  'target -bwlimit'

Use of uninitialized value in concatenation (.) or string at ./syncoid line 1317.
just an asuumption on my part considering the rest of the variables with same name dont have the space in that location
---
 syncoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncoid b/syncoid
index caaba44..ec40733 100755
--- a/syncoid
+++ b/syncoid
@@ -1314,7 +1314,7 @@ sub buildsynccmd {
 
 		my $remotecmd = "";
 		if ($directmbuffer) {
-			$remotecmd .= " $mbuffercmd $args{'target -bwlimit'} -W $directtimeout -I " . $directlisten . " $mbufferoptions |";
+			$remotecmd .= " $mbuffercmd $args{'target-bwlimit'} -W $directtimeout -I " . $directlisten . " $mbufferoptions |";
 		} elsif (length $directlisten) {
 			$remotecmd .= " busybox nc -l " . $directlisten . " -w $directtimeout |";
 		}

From 3bae6baf3d68efcc451d2db366c4692c854b3677 Mon Sep 17 00:00:00 2001
From: rbike <46862457+rbike@users.noreply.github.com>
Date: Tue, 2 Mar 2021 16:03:16 +0100
Subject: [PATCH 08/22] syncoid hold

Added hold feature to syncoid.
Parameter "--use-hold" sets hold for newsyncsmap and removes hold from matchingsnap both on source and target. Hold name is "syncoid" + identiifer + host name
---
 syncoid | 41 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/syncoid b/syncoid
index b771d8f..22d1c65 100755
--- a/syncoid
+++ b/syncoid
@@ -3,6 +3,9 @@
 # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
+# 
+# 2021-03-ß2: redbike
+# support f0r zfs holds added
 
 $::VERSION = '2.0.3';
 
@@ -25,7 +28,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback",
-                   "create-bookmark", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize",
+                   "create-bookmark", "use-hold", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize",
                    "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -355,6 +358,7 @@ sub syncdataset {
 	}
 
 	my $newsyncsnap;
+	my $matchingsnap;
 
 	# skip snapshot checking/creation in case of resumed receive
 	if (!defined($receivetoken)) {
@@ -602,7 +606,7 @@ sub syncdataset {
 		my $bookmark = 0;
 		my $bookmarkcreation = 0;
 
-		my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
+		$matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
 		if (! $matchingsnap) {
 			# no matching snapshots, check for bookmarks as fallback
 			my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
@@ -827,7 +831,36 @@ sub syncdataset {
 			#setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly);
 		}
 	}
-
+# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target
+	# hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed
+	if (defined $args{'use-hold'}) {
+		my $holdcmd;
+		my $holdreleasecmd;
+		my $hostid = hostname();
+		my $matchingsnapescaped = escapeshellparam($matchingsnap);
+		my $holdname = "syncoid\_$identifier$hostid";
+		if ($sourcehost ne '') {
+			$holdcmd = "$sshcmd $sourcehost " . escapeshellparam("$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped");
+			$holdreleasecmd = "$sshcmd $sourcehost " . escapeshellparam("$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped");
+		} else {
+			$holdcmd = "$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped";
+			$holdreleasecmd = "$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped";
+		};
+		if ($debug) { print "DEBUG: Set new hold on source: $holdcmd\n"; }
+		system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?";
+		if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; }
+		system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+		if ($targethost ne '') {
+			$holdcmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped");
+			$holdreleasecmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped");
+		} else {
+			$holdcmd = "$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped";				$holdreleasecmd = "$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped";
+		};
+		if ($debug) { print "DEBUG: Set new hold on target: $holdcmd\n"; }
+		system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?";
+		if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; }
+		system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+	}
 	if (defined $args{'no-sync-snap'}) {
 		if (defined $args{'create-bookmark'}) {
 			my $bookmarkcmd;
@@ -1325,7 +1358,7 @@ sub buildsynccmd {
 sub pruneoldsyncsnaps {
 	my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_;
 
-	my $fsescaped = escapeshellparam($fs);
+	my $fsescaped = escapeshellparam($fs);	
 
 	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
 

From 2858fbe907680f04f0cd449bf676a2b33e13343b Mon Sep 17 00:00:00 2001
From: rbike <46862457+rbike@users.noreply.github.com>
Date: Wed, 3 Mar 2021 13:49:13 +0100
Subject: [PATCH 09/22] Docs update + error handling

updated in-program docs and readme,
check if matchingsnap exists before releasing hold
---
 README.md |  5 ++++-
 syncoid   | 23 +++++++++++++++--------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index b4558c0..3048b7f 100644
--- a/README.md
+++ b/README.md
@@ -272,7 +272,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 + --identifier=
 
-	Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . .
+	Adds the given identifier to the snapshot and hold name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . .
 
 + -r --recursive
 
@@ -314,6 +314,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to create a zfs bookmark for the newest snapshot after it got replicated successfully. The bookmark name will be equal to the snapshot name. Only works in combination with the --no-sync-snap option. This can be very useful for irregular replication where the last matching snapshot on the source was already deleted but the bookmark remains so a replication is still possible.
 
++ --use-hold
+	This argument tells syncoid to add a hold to the newest snapshot on the source and target after replication succeeds and to remove the hold after the next succesful replication. Setting a hold prevents the snapshots from being destroyed. The hold name incldues the identifier if set. This allows for separate holds in case of replication to multiple targets.
+
 + --preserve-recordsize
 
 	This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends.
diff --git a/syncoid b/syncoid
index 22d1c65..84510fb 100755
--- a/syncoid
+++ b/syncoid
@@ -4,8 +4,8 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 # 
-# 2021-03-ß2: redbike
-# support f0r zfs holds added
+# 2021-03-03: redbike
+# support for zfs holds added
 
 $::VERSION = '2.0.3';
 
@@ -845,21 +845,27 @@ sub syncdataset {
 		} else {
 			$holdcmd = "$sourcesudocmd $zfscmd hold $holdname $sourcefsescaped\@$newsyncsnapescaped";
 			$holdreleasecmd = "$sourcesudocmd $zfscmd release $holdname $sourcefsescaped\@$matchingsnapescaped";
-		};
+		}
 		if ($debug) { print "DEBUG: Set new hold on source: $holdcmd\n"; }
 		system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?";
-		if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; }
-		system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+		# Do hold release only if matchingsnap exists
+		if ($matchingsnap) {
+			if ($debug) { print "DEBUG: Release old hold on source: $holdreleasecmd\n"; }
+			system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+		}
 		if ($targethost ne '') {
 			$holdcmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped");
 			$holdreleasecmd = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped");
 		} else {
 			$holdcmd = "$targetsudocmd $zfscmd hold $holdname $targetfsescaped\@$newsyncsnapescaped";				$holdreleasecmd = "$targetsudocmd $zfscmd release $holdname $targetfsescaped\@$matchingsnapescaped";
-		};
+		}
 		if ($debug) { print "DEBUG: Set new hold on target: $holdcmd\n"; }
 		system($holdcmd) == 0 or warn "WARNING: $holdcmd failed: $?";
-		if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; }
-		system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+		# Do hold release only if matchingsnap exists
+		if ($matchingsnap) {
+			if ($debug) { print "DEBUG: Release old hold on target: $holdreleasecmd\n"; }
+			system($holdreleasecmd) == 0 or warn "WARNING: $holdreleasecmd failed: $?";
+		}
 	}
 	if (defined $args{'no-sync-snap'}) {
 		if (defined $args{'create-bookmark'}) {
@@ -2005,6 +2011,7 @@ Options:
   --no-sync-snap        Does not create new snapshot, only transfers existing
   --keep-sync-snap      Don't destroy created sync snapshots
   --create-bookmark     Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap)
+  --use-hold            Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets
   --preserve-recordsize Preserves the recordsize on initial sends to the target
   --no-clone-rollback   Does not rollback clones on target
   --no-rollback         Does not rollback clones or snapshots on target (it probably requires a readonly target)

From 21eee41fdb20a6bc3b2068c18a3061723f52435a Mon Sep 17 00:00:00 2001
From: rbike <46862457+rbike@users.noreply.github.com>
Date: Thu, 1 Apr 2021 09:23:48 +0200
Subject: [PATCH 10/22] Final revision

comments from @phreaker0 implemented
---
 syncoid | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/syncoid b/syncoid
index 84510fb..a3708c6 100755
--- a/syncoid
+++ b/syncoid
@@ -3,9 +3,6 @@
 # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
-# 
-# 2021-03-03: redbike
-# support for zfs holds added
 
 $::VERSION = '2.0.3';
 
@@ -831,7 +828,7 @@ sub syncdataset {
 			#setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly);
 		}
 	}
-# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target
+	# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target
 	# hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed
 	if (defined $args{'use-hold'}) {
 		my $holdcmd;
@@ -1364,7 +1361,7 @@ sub buildsynccmd {
 sub pruneoldsyncsnaps {
 	my ($rhost,$fs,$newsyncsnap,$isroot,@snaps) = @_;
 
-	my $fsescaped = escapeshellparam($fs);	
+	my $fsescaped = escapeshellparam($fs);
 
 	if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
 

From 59c59e2b000553c439fa9785aeb17d5d1822adaa Mon Sep 17 00:00:00 2001
From: Mark Scholes <mark.scholes@gmail.com>
Date: Thu, 19 May 2022 11:54:16 +0100
Subject: [PATCH 11/22] Removed unneeded 2>/dev/null from mbufferoptions

---
 syncoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncoid b/syncoid
index ec6ae9d..ac8f74f 100755
--- a/syncoid
+++ b/syncoid
@@ -97,7 +97,7 @@ my $pscmd = 'ps';
 my $pvcmd = 'pv';
 my $mbuffercmd = 'mbuffer';
 my $sudocmd = 'sudo';
-my $mbufferoptions = "-q -s 128k -m $mbuffer_size 2>/dev/null";
+my $mbufferoptions = "-q -s 128k -m $mbuffer_size";
 # currently using POSIX compatible command to check for program existence because we aren't depending on perl
 # being present on remote machines.
 my $checkcmd = 'command -v';

From a7f9f08f4789108429d016d484bb3d62b873eb54 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 21 Mar 2023 16:44:38 +0100
Subject: [PATCH 12/22] fixes some tests which fail on fast storage

---
 tests/syncoid/5_reset_resume_state/run.sh  | 3 +++
 tests/syncoid/6_reset_resume_state2/run.sh | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/tests/syncoid/5_reset_resume_state/run.sh b/tests/syncoid/5_reset_resume_state/run.sh
index 6e71002..43ec78f 100755
--- a/tests/syncoid/5_reset_resume_state/run.sh
+++ b/tests/syncoid/5_reset_resume_state/run.sh
@@ -45,6 +45,9 @@ wait
 sleep 1
 
 ../../../syncoid --debug --compress=none --no-resume "${POOL_NAME}"/src "${POOL_NAME}"/dst | grep "reset partial receive state of syncoid"
+
+sleep 1
+
 ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
 
 exit $?
diff --git a/tests/syncoid/6_reset_resume_state2/run.sh b/tests/syncoid/6_reset_resume_state2/run.sh
index 1afc921..d05696b 100755
--- a/tests/syncoid/6_reset_resume_state2/run.sh
+++ b/tests/syncoid/6_reset_resume_state2/run.sh
@@ -47,6 +47,9 @@ sleep 1
 
 zfs destroy "${POOL_NAME}"/src@big
 ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst # | grep "reset partial receive state of syncoid"
+
+sleep 1
+
 ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
 
 exit $?

From 91d96a4c8739d4955983153b571cccb0bacbff81 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Fri, 24 Mar 2023 09:00:07 +0100
Subject: [PATCH 13/22] support bookmarks which are taken in the same second

---
 syncoid | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/syncoid b/syncoid
index e5046f3..af9518e 100755
--- a/syncoid
+++ b/syncoid
@@ -25,7 +25,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "source-bwlimit=s", "target-bwlimit=s", "sshconfig=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark",
-                   "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size) 
+                   "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size)
                    or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -1729,6 +1729,7 @@ sub getbookmarks() {
 	# as though each were an entirely separate get command.
 
 	my $lastguid;
+	my %creationtimes=();
 
 	foreach my $line (@rawbookmarks) {
 		# only import bookmark guids, creation from the specified filesystem
@@ -1745,7 +1746,24 @@ sub getbookmarks() {
 			$creation =~ s/^.*\tcreation\t*(\d*).*/$1/;
 			my $bookmark = $line;
 			$bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/;
-			$bookmarks{$lastguid}{'creation'}=$creation . "000";
+
+			# the accuracy of the creation timestamp is only for a second, but
+			# bookmarks in the same second are possible. The list command
+			# has an ordered output so we append another three digit running number
+			# to the creation timestamp and make sure those are ordered correctly
+			# for bookmarks with the same creation timestamp
+			my $counter = 0;
+			my $creationsuffix;
+			while ($counter < 999) {
+				$creationsuffix = sprintf("%s%03d", $creation, $counter);
+				if (!defined $creationtimes{$creationsuffix}) {
+					$creationtimes{$creationsuffix} = 1;
+					last;
+				}
+				$counter += 1;
+			}
+
+			$bookmarks{$lastguid}{'creation'}=$creationsuffix;
 		}
 	}
 

From 2d89434ac3181cd7e8d1b2f9d5fe676d85eadd17 Mon Sep 17 00:00:00 2001
From: Mathieu Arnold <mat@mat.cc>
Date: Fri, 13 Mar 2020 14:20:25 +0100
Subject: [PATCH 14/22] Add target snapshot deletion.

---
 README.md |  3 +++
 syncoid   | 18 +++++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1cb2f17..51ac568 100644
--- a/README.md
+++ b/README.md
@@ -319,6 +319,9 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 + --preserve-recordsize
 
 	This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends.
++ --push-snap-removal
+
+	With this argument snapshots that are removed in the source will also be removed.  Use this if you only want to handle snapshots on the source.
 
 + --no-clone-rollback
 
diff --git a/syncoid b/syncoid
index e5046f3..3796ce1 100755
--- a/syncoid
+++ b/syncoid
@@ -25,7 +25,8 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "source-bwlimit=s", "target-bwlimit=s", "sshconfig=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark",
-                   "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size) 
+                   "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size,
+		   "push-snap-removal",) 
                    or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -888,6 +889,20 @@ sub syncdataset {
 		}
 	}
 
+	if (defined $args{'push-snap-removal'}) {
+		foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) {
+			if (!exists $snaps{'source'}{$snap}) {
+				if ($targethost ne '') {
+					if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; }
+					system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap"));
+				} else {
+					if ($debug) { print "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; }
+					system ("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap");
+				}
+			}
+		}
+	}
+
 } # end syncdataset()
 
 sub compressargset {
@@ -1999,6 +2014,7 @@ Options:
   --create-bookmark     Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap)
   --preserve-recordsize Preserves the recordsize on initial sends to the target
   --no-rollback         Does not rollback snapshots on target (it probably requires a readonly target)
+  --push-snap-removal   Remove snapshots on the target that do not exist on the source any more
   --exclude=REGEX       Exclude specific datasets which match the given regular expression. Can be specified multiple times
   --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ...
   --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ...

From 63dd819ec55d181283e184e898a88bc3751e7e89 Mon Sep 17 00:00:00 2001
From: Mathieu Arnold <mat@mat.cc>
Date: Mon, 11 May 2020 16:37:40 +0200
Subject: [PATCH 15/22] Rename option.

---
 README.md | 5 +++--
 syncoid   | 6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 51ac568..f6028b0 100644
--- a/README.md
+++ b/README.md
@@ -319,9 +319,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 + --preserve-recordsize
 
 	This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends.
-+ --push-snap-removal
 
-	With this argument snapshots that are removed in the source will also be removed.  Use this if you only want to handle snapshots on the source.
++ --delete-target-snapshots
+
+	With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
 
 + --no-clone-rollback
 
diff --git a/syncoid b/syncoid
index 3796ce1..19c8656 100755
--- a/syncoid
+++ b/syncoid
@@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark",
                    "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size,
-		   "push-snap-removal",) 
+                   "delete-target-snapshots") 
                    or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -889,7 +889,7 @@ sub syncdataset {
 		}
 	}
 
-	if (defined $args{'push-snap-removal'}) {
+	if (defined $args{'delete-target-snapshots'}) {
 		foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) {
 			if (!exists $snaps{'source'}{$snap}) {
 				if ($targethost ne '') {
@@ -2014,7 +2014,7 @@ Options:
   --create-bookmark     Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap)
   --preserve-recordsize Preserves the recordsize on initial sends to the target
   --no-rollback         Does not rollback snapshots on target (it probably requires a readonly target)
-  --push-snap-removal   Remove snapshots on the target that do not exist on the source any more
+  --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
   --exclude=REGEX       Exclude specific datasets which match the given regular expression. Can be specified multiple times
   --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ...
   --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ...

From ecd14005395059e5dbbbdd8f7530d479d1e5843b Mon Sep 17 00:00:00 2001
From: Mathieu Arnold <mat@mat.cc>
Date: Mon, 17 Aug 2020 16:36:14 +0200
Subject: [PATCH 16/22] Handle output/errors of those zfs destroy commands.

If there was an obsolete remote syncoid_hostname_* snapshot that did not
get removed at the correct time, for some reason, like, maybe, network
problems, it would have been cleaned up in pruneoldsyncsnaps just before
this code, and we would get a strange error message saying:

    could not find any snapshots to destroy; check snapshot names.

Also, when using --quiet, do not output anything, as failing to remove
an obsolete snapshot is not really a big problem.
---
 syncoid | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/syncoid b/syncoid
index 19c8656..4393e4f 100755
--- a/syncoid
+++ b/syncoid
@@ -890,15 +890,18 @@ sub syncdataset {
 	}
 
 	if (defined $args{'delete-target-snapshots'}) {
-		foreach my $snap ( sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } keys %{ $snaps{'target'} }) {
-			if (!exists $snaps{'source'}{$snap}) {
-				if ($targethost ne '') {
-					if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; }
-					system ("$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap"));
-				} else {
-					if ($debug) { print "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap\n"; }
-					system ("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snap");
-				}
+		my $snaps = join ',', grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
+		if ($snaps ne '') {
+			my $command;
+			if ($targethost ne '') {
+				$command = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps");
+			} else {
+				$command = "$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps";
+			}
+			if ($debug) { print "$command\n"; }
+			my ($stdout, $stderr, $result) = capture { system $command; };
+			if ($result != 0 && !$quiet) {
+				warn "$command failed: $stderr";
 			}
 		}
 	}

From f711e6bf28e8b10f9e0f91f81f93ff6149c8492d Mon Sep 17 00:00:00 2001
From: Mathieu Arnold <mat@mat.cc>
Date: Mon, 17 May 2021 13:32:22 +0200
Subject: [PATCH 17/22] Add a note about when snapshot deletion is done.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index f6028b0..5ad2e0c 100644
--- a/README.md
+++ b/README.md
@@ -323,6 +323,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 + --delete-target-snapshots
 
 	With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
+	Note that snapshot deletion is only done after a successful synchronization. If no new snapshots are found, no synchronization is done and no deletion either.
 
 + --no-clone-rollback
 

From 2f706a4ae1cab80417e76cc071aa1c561706791d Mon Sep 17 00:00:00 2001
From: Mathieu Arnold <mat@mat.cc>
Date: Sat, 8 Apr 2023 09:58:40 +0200
Subject: [PATCH 18/22] Batch snapshot deletion.

This is to prevent a problem with a large amount of snapshots which
exceed the allowed shell command length.
---
 syncoid | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/syncoid b/syncoid
index 4393e4f..1736afe 100755
--- a/syncoid
+++ b/syncoid
@@ -890,8 +890,14 @@ sub syncdataset {
 	}
 
 	if (defined $args{'delete-target-snapshots'}) {
-		my $snaps = join ',', grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
-		if ($snaps ne '') {
+		# Find the snapshots that exist on the target, filter with
+		# those that exist on the source. Remaining are the snapshots
+		# that are only on the target. Then sort by creation date, as
+		# to remove the oldest snapshots first.
+		my @to_delete = sort { $snaps{'target'}{$a}{'creation'}<=>$snaps{'target'}{$b}{'creation'} } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
+		while (@to_delete) {
+			# Create batch of snapshots to remove
+			my $snaps = join ',', splice(@to_delete, 0, 50);
 			my $command;
 			if ($targethost ne '') {
 				$command = "$sshcmd $targethost " . escapeshellparam("$targetsudocmd $zfscmd destroy $targetfsescaped\@$snaps");

From cc495183afb80d4803aa0d7717dbae0b49688274 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Wed, 26 Apr 2023 19:47:49 +0200
Subject: [PATCH 19/22] fix default behaviour if run without providing
 arguments

---
 sanoid | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanoid b/sanoid
index 6de6c30..12a184a 100755
--- a/sanoid
+++ b/sanoid
@@ -30,7 +30,7 @@ GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet",
           ) or pod2usage(2);
 
 # If only config directory (or nothing) has been specified, default to --cron --verbose
-if (keys %args < 2) {
+if (keys %args < 4) {
 	$args{'cron'} = 1;
 	$args{'verbose'} = 1;
 }

From f3d4d309b5a6ccba11512b698041e80a0c8518fa Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 18 Jul 2023 08:38:40 +0200
Subject: [PATCH 20/22] implemented flag for preserving properties without the
 zfs -p flag

---
 README.md                                  |  4 ++
 syncoid                                    | 64 ++++++++++++++++++++-
 tests/syncoid/9_preserve_properties/run.sh | 66 ++++++++++++++++++++++
 3 files changed, 131 insertions(+), 3 deletions(-)
 create mode 100755 tests/syncoid/9_preserve_properties/run.sh

diff --git a/README.md b/README.md
index 7a10dac..6549617 100644
--- a/README.md
+++ b/README.md
@@ -323,6 +323,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends.
 
++ --preserve-properties
+
+	This argument tells syncoid to get all locally set dataset properties from the source and apply all supported ones on the target before writing any data. It's similar to the '-p' flag for zfs send but also works for encrypted datasets in non raw sends. This only applies to initial sends.
+
 + --delete-target-snapshots
 
 	With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
diff --git a/syncoid b/syncoid
index eb38539..6cde9f9 100755
--- a/syncoid
+++ b/syncoid
@@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "use-hold",
                    "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size,
-                   "delete-target-snapshots", "insecure-direct-connection=s")
+                   "delete-target-snapshots", "insecure-direct-connection=s", "preserve-properties")
                    or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -487,11 +487,19 @@ sub syncdataset {
 		}
 		my $oldestsnapescaped = escapeshellparam($oldestsnap);
 
-		if (defined $args{'preserve-recordsize'}) {
+		if (defined $args{'preserve-properties'}) {
+			my %properties = getlocalzfsvalues($sourcehost,$sourcefs,$sourceisroot);
+
+			foreach my $key (keys %properties) {
+				my $value = $properties{$key};
+				if ($debug) { print "DEBUG: will set $key to $value ...\n"; }
+				$recvoptions .= " -o $key=$value";
+			}
+		} elsif (defined $args{'preserve-recordsize'}) {
 			my $type = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'type');
 			if ($type eq "filesystem") {
 				my $recordsize = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'recordsize');
-				$recvoptions .= "-o recordsize=$recordsize"
+				$recvoptions .= "-o recordsize=$recordsize";
 			}
 		}
 
@@ -1335,6 +1343,55 @@ sub getzfsvalue {
 	return $wantarray ? ($value, $error) : $value;
 }
 
+sub getlocalzfsvalues {
+	my ($rhost,$fs,$isroot) = @_;
+
+	my $fsescaped = escapeshellparam($fs);
+
+	if ($rhost ne '') {
+		$rhost = "$sshcmd $rhost";
+		# double escaping needed
+		$fsescaped = escapeshellparam($fsescaped);
+	}
+
+	if ($debug) { print "DEBUG: getting locally set values of properties on $fs...\n"; }
+	my $mysudocmd;
+	if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; }
+	if ($debug) { print "$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped\n"; }
+	my ($values, $error, $exit) = capture {
+		system("$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped");
+	};
+
+	my %properties=();
+
+	if ($exit != 0) {
+		warn "WARNING: getlocalzfsvalues failed for $fs: $error";
+		if ($exitcode < 1) { $exitcode = 1; }
+		return %properties;
+	}
+
+	my @blacklist = (
+		"available", "compressratio", "createtxg", "creation", "clones",
+		"defer_destroy", "encryptionroot", "filesystem_count", "keystatus", "guid",
+		"logicalreferenced", "logicalused", "mounted", "objsetid", "origin",
+		"receive_resume_token", "redact_snaps", "referenced", "refcompressratio", "snapshot_count",
+		"type", "used", "usedbychildren", "usedbydataset", "usedbyrefreservation",
+		"usedbysnapshots", "userrefs", "snapshots_changed", "volblocksize", "written",
+		"version", "volsize", "casesensitivity", "normalization", "utf8only"
+	);
+	my %blacklisthash = map {$_ => 1} @blacklist;
+
+	foreach (split(/\n/,$values)) {
+		my @parts = split(/\t/, $_);
+		if (exists $blacklisthash{$parts[1]}) {
+			next;
+		}
+		$properties{$parts[1]} = $parts[2];
+	}
+
+	return %properties;
+}
+
 sub readablebytes {
 	my $bytes = shift;
 	my $disp;
@@ -2153,6 +2210,7 @@ Options:
   --create-bookmark     Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap)
   --use-hold            Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets
   --preserve-recordsize Preserves the recordsize on initial sends to the target
+  --preserve-properties Preserves locally set dataset properties similiar to the zfs send -p flag but this one will also work for encrypted datasets in non raw sends
   --no-rollback         Does not rollback snapshots on target (it probably requires a readonly target)
   --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
   --exclude=REGEX       Exclude specific datasets which match the given regular expression. Can be specified multiple times
diff --git a/tests/syncoid/9_preserve_properties/run.sh b/tests/syncoid/9_preserve_properties/run.sh
new file mode 100755
index 0000000..497ce9a
--- /dev/null
+++ b/tests/syncoid/9_preserve_properties/run.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# test preserving locally set properties from the src dataset to the target one
+
+set -x
+set -e
+
+. ../../common/lib.sh
+
+POOL_IMAGE="/tmp/syncoid-test-9.zpool"
+MOUNT_TARGET="/tmp/syncoid-test-9.mount"
+POOL_SIZE="1000M"
+POOL_NAME="syncoid-test-9"
+
+truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
+
+zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
+
+function cleanUp {
+  zpool export "${POOL_NAME}"
+}
+
+# export pool in any case
+trap cleanUp EXIT
+
+zfs create -o recordsize=16k -o xattr=on -o mountpoint=none -o primarycache=none "${POOL_NAME}"/src
+zfs create -V 100M -o volblocksize=8k "${POOL_NAME}"/src/zvol8
+zfs create -V 100M -o volblocksize=16k -o primarycache=all "${POOL_NAME}"/src/zvol16
+zfs create -V 100M -o volblocksize=64k "${POOL_NAME}"/src/zvol64
+zfs create -o recordsize=16k -o primarycache=none "${POOL_NAME}"/src/16
+zfs create -o recordsize=32k -o acltype=posixacl "${POOL_NAME}"/src/32
+
+../../../syncoid --preserve-properties --recursive --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
+
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst)" != "16K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get mountpoint -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get xattr -H -o value -t filesystem "${POOL_NAME}"/dst)" != "on" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "16K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "32K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get acltype -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "posix" ]; then
+	exit 1
+fi

From 538416879d45baa981307aeadd3abe34adae3874 Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 18 Jul 2023 18:09:51 +0200
Subject: [PATCH 21/22] prepare 2.2.0

---
 CHANGELIST                | 14 ++++++++++++++
 VERSION                   |  2 +-
 findoid                   |  2 +-
 packages/debian/changelog | 18 ++++++++++++++++++
 packages/rhel/sanoid.spec |  4 +++-
 sanoid                    |  2 +-
 syncoid                   |  2 +-
 7 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/CHANGELIST b/CHANGELIST
index 6ddc565..314e431 100644
--- a/CHANGELIST
+++ b/CHANGELIST
@@ -1,3 +1,17 @@
+2.2.0  [overall]   documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0)
+       [syncoid]   implemented flag for preserving properties without the zfs -p flag (@phreaker0)
+       [syncoid]   implemented target snapshot deletion (@mat813)
+       [syncoid]   support bookmarks which are taken in the same second (@delxg, @phreaker0)
+       [syncoid]   exit with an error if the specified src dataset doesn't exist (@phreaker0)
+       [syncoid]   rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0)
+       [syncoid]   append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3)
+       [syncoid]   implemented support for ssh_config(5) files (@endreszabo)
+       [syncoid]   snapshot hold/unhold support (@rbike)
+       [sanoid]    handle duplicate key definitions gracefully (@phreaker0)
+       [syncoid]   implemented removal of conflicting snapshots with force-delete option (@phreaker0)
+       [sanoid]    implemented pre pruning script hook (@phreaker0)
+       [syncoid]   implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0)
+
 2.1.0  [overall]   documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0)
        [syncoid]   do not require user to be specified for syncoid (@aerusso)
        [syncoid]   implemented option for keeping sync snaps (@phreaker0)
diff --git a/VERSION b/VERSION
index 7ec1d6d..ccbccc3 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.0
+2.2.0
diff --git a/findoid b/findoid
index 98ad581..0bb5e5f 100755
--- a/findoid
+++ b/findoid
@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 
 use strict;
 use warnings;
diff --git a/packages/debian/changelog b/packages/debian/changelog
index 4cab69b..00c0c07 100644
--- a/packages/debian/changelog
+++ b/packages/debian/changelog
@@ -1,3 +1,21 @@
+sanoid (2.2.0) unstable; urgency=medium
+
+  [overall]   documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0)
+  [syncoid]   implemented flag for preserving properties without the zfs -p flag (@phreaker0)
+  [syncoid]   implemented target snapshot deletion (@mat813)
+  [syncoid]   support bookmarks which are taken in the same second (@delxg, @phreaker0)
+  [syncoid]   exit with an error if the specified src dataset doesn't exist (@phreaker0)
+  [syncoid]   rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0)
+  [syncoid]   append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3)
+  [syncoid]   implemented support for ssh_config(5) files (@endreszabo)
+  [syncoid]   snapshot hold/unhold support (@rbike)
+  [sanoid]    handle duplicate key definitions gracefully (@phreaker0)
+  [syncoid]   implemented removal of conflicting snapshots with force-delete option (@phreaker0)
+  [sanoid]    implemented pre pruning script hook (@phreaker0)
+  [syncoid]   implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0)
+
+ -- Jim Salter <github@jrs-s.net>  Tue, 18 Jul 2023 10:04:00 +0200
+
 sanoid (2.1.0) unstable; urgency=medium
 
   [overall]   documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0)
diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec
index 376f58a..218f52d 100644
--- a/packages/rhel/sanoid.spec
+++ b/packages/rhel/sanoid.spec
@@ -1,4 +1,4 @@
-%global version 2.1.0
+%global version 2.2.0
 %global git_tag v%{version}
 
 # Enable with systemctl "enable sanoid.timer"
@@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name}
 %endif
 
 %changelog
+* Tue Jul 18 2023 Christoph Klaffl <christoph@phreaker.eu> - 2.2.0
+- Bump to 2.2.0
 * Tue Nov 24 2020 Christoph Klaffl <christoph@phreaker.eu> - 2.1.0
 - Bump to 2.1.0
 * Wed Oct 02 2019 Christoph Klaffl <christoph@phreaker.eu> - 2.0.3
diff --git a/sanoid b/sanoid
index 12a184a..5150f3b 100755
--- a/sanoid
+++ b/sanoid
@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 my $MINIMUM_DEFAULTS_VERSION = 2;
 
 use strict;
diff --git a/syncoid b/syncoid
index 6cde9f9..5ed00f0 100755
--- a/syncoid
+++ b/syncoid
@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 
 use strict;
 use warnings;

From 7ab0ac0cf6bdc112b2b0a5a93f464892af66ca8a Mon Sep 17 00:00:00 2001
From: Christoph Klaffl <christoph@phreaker.eu>
Date: Tue, 25 Jul 2023 22:56:25 +0200
Subject: [PATCH 22/22] trim config values

---
 sanoid | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sanoid b/sanoid
index 5150f3b..8848e3b 100755
--- a/sanoid
+++ b/sanoid
@@ -905,6 +905,8 @@ sub init {
 				warn "duplicate key '$key' in section '$section', using the value from the first occurence and ignoring the others.\n";
 				$ini{$section}{$key} = $value->[0];
 			}
+			# trim
+			$ini{$section}{$key} =~ s/^\s+|\s+$//g;
 		}
 
 		if ($section =~ /^template_/) { next; } # don't process templates directly