From c0c30500765395cd22f975abb198ce3d4fb1d1f3 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:21:06 +0200 Subject: [PATCH 1/7] added option for skipping the parent dataset in recursive replication --- README.md | 4 ++++ syncoid | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc75bdf..7392fa7 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This will also transfer child datasets. ++ --skip-parent + + This will skip the syncing of the parent dataset. Does nothing without '--recursive' option. + + --compress Currently accepted options: gzip, pigz-fast, pigz-slow, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. diff --git a/syncoid b/syncoid index 999ee79..5cbc1c4 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "skip-parent") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -141,6 +141,11 @@ sub getchilddatasets { my @children = ; close FH; + if (defined $args{'skip-parent'}) { + # parent dataset is the first element + shift @children; + } + return @children; } @@ -1137,6 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets + --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From 70b259ac3cb9ef9167776eac0424d0a78ea0c0c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 18:24:34 +0200 Subject: [PATCH 2/7] typos --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 5cbc1c4..8ab3a41 100755 --- a/syncoid +++ b/syncoid @@ -1142,7 +1142,7 @@ Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none --recursive|r Also transfers child datasets - --skip-parent Skipp the syncing of the parent dataset. Doesg nothing without '--recursive' option. + --skip-parent Skips syncing of the parent dataset. Does nothing without '--recursive' option. --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer --no-stream Replicates using newest snapshot instead of intermediates From 0f8fee7637ec1265bd0a72011a08b0666e6123c4 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 19 Jun 2018 19:43:36 +0200 Subject: [PATCH 3/7] added option for using extra identification in the snapshot name for replicating to multiple targets --- README.md | 4 ++++ syncoid | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc75bdf..632a0af 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This is the destination dataset. It can be either local or remote. ++ --identifier= + + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + + -r --recursive This will also transfer child datasets. diff --git a/syncoid b/syncoid index 999ee79..dcb9e2f 100755 --- a/syncoid +++ b/syncoid @@ -19,7 +19,7 @@ use Sys::Hostname; my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", - "debug", "quiet", "no-stream", "no-sync-snap", "no-resume") or pod2usage(2); + "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "identifier=s") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -71,6 +71,17 @@ if (length $args{'sshkey'}) { } my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref required +my $identifier = ""; +if (length $args{'identifier'}) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + # invalid extra identifier + print("CRITICAL: extra identifier contains invalid chars!\n"); + pod2usage(2); + exit 127; + } + $identifier = "$args{'identifier'}_"; +} + # figure out if source and/or target are remote. $sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } @@ -812,7 +823,7 @@ sub pruneoldsyncsnaps { # only prune snaps beginning with syncoid and our own hostname foreach my $snap(@snaps) { - if ($snap =~ /^syncoid_\Q$hostid\E/) { + if ($snap =~ /^syncoid_\Q$identifier$hostid\E/) { # no matter what, we categorically refuse to # prune the new sync snap we created for this run if ($snap ne $newsyncsnap) { @@ -898,7 +909,7 @@ sub newsyncsnap { if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } my $hostid = hostname(); my %date = getdate(); - my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; + my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n"; system($snapcmd) == 0 or die "CRITICAL ERROR: $snapcmd failed: $?"; @@ -1136,6 +1147,7 @@ syncoid - ZFS snapshot replication tool Options: --compress=FORMAT Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, lzo (default) & none + --identifier=EXTRA Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets. --recursive|r Also transfers child datasets --source-bwlimit= Bandwidth limit on the source transfer --target-bwlimit= Bandwidth limit on the target transfer From ba3836ec520efc30689238e294de1d1c2026fc1b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 6 Jul 2018 15:52:54 +0200 Subject: [PATCH 4/7] fixed monitor-health command for pools containing cache and log devices --- sanoid | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 9cd9d33..485ee08 100755 --- a/sanoid +++ b/sanoid @@ -976,6 +976,11 @@ sub check_zpool() { ## other cases my ($dev, $sta) = /^\s+(\S+)\s+(\S+)/; + if (!defined($sta)) { + # cache and logs are special and don't have a status + next; + } + ## pool online, not degraded thanks to dead/corrupted disk if ($state eq "OK" && $sta eq "UNAVAIL") { $state="WARNING"; @@ -1111,7 +1116,7 @@ sub checklock { # make sure lockfile contains something if ( -z $lockfile) { # zero size lockfile, something is wrong - die "ERROR: something is wrong! $lockfile is empty\n"; + die "ERROR: something is wrong! $lockfile is empty\n"; } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if From f9c1cbb74a3c07fc9e0368721cdcedcc0b2b4a0f Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sat, 7 Jul 2018 12:06:35 -0400 Subject: [PATCH 5/7] Update INSTALL --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 33b510d..f0de17b 100644 --- a/INSTALL +++ b/INSTALL @@ -8,7 +8,7 @@ default for SSH transport since v1.4.6. Syncoid runs will fail if one of them is not available on either end of the transport. On Ubuntu: apt install pv lzop mbuffer -On CentOS: yum install lzo pv mbuffer lzop +On CentOS: yum install lzo pv mbuffer lzop perl-Data-Dumper On FreeBSD: pkg install pv mbuffer lzop FreeBSD notes: FreeBSD may place pv and lzop in somewhere other than From dc0afebb30865faa16f42c496b2c59e5ed72b690 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:06:17 +0200 Subject: [PATCH 6/7] allow extra identifier to contain all characters for snapshots names which are allowed by zfs --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index d25d9de..aaf5490 100755 --- a/syncoid +++ b/syncoid @@ -73,7 +73,7 @@ my $sshoptions = join " ", map { "-o " . $_ } @{$args{'sshoption'}}; # deref req my $identifier = ""; if (length $args{'identifier'}) { - if ($args{'identifier'} !~ /^[a-zA-Z0-9-]+$/) { + if ($args{'identifier'} !~ /^[a-zA-Z0-9-_:.]+$/) { # invalid extra identifier print("CRITICAL: extra identifier contains invalid chars!\n"); pod2usage(2); From f409b955694b9dc5f403f16593cc6540cac7fe67 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 7 Jul 2018 20:10:02 +0200 Subject: [PATCH 7/7] updated parameter documention of --identifier regarding allowed characters --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1f651..e359886 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --identifier= - Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9 and -. + Adds the given identifier to the snapshot name after "syncoid_" prefix and before the hostname. This enables the use case of reliable replication to multiple targets from the same host. The following chars are allowed: a-z, A-Z, 0-9, _, -, : and . . + -r --recursive