From 5d7ee404058acdfb207316ba88417bf5fcf68d62 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 24 May 2019 08:29:54 +0200 Subject: [PATCH 01/42] fix centos package name --- INSTALL.md | 2 +- packages/rhel/sanoid.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index c90d5d1..8f0af5d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -51,7 +51,7 @@ Install prerequisite software: # Install and enable epel if we don't already have it, and git too sudo yum install -y epel-release git # Install the packages that Sanoid depends on: -sudo yum install -y perl-Config-IniFiles perl-Data-Dumper perl-capture-tiny lzop mbuffer mhash pv +sudo yum install -y perl-Config-IniFiles perl-Data-Dumper perl-Capture-Tiny lzop mbuffer mhash pv ``` Clone this repo, then put the executables and config files into the appropriate directories: diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index c605be9..8ce0360 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -14,7 +14,7 @@ License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid Source0: https://github.com/jimsalterjrs/%{name}/archive/%{git_tag}/%{name}-%{version}.tar.gz -Requires: perl, mbuffer, lzop, pv, perl-Config-IniFiles, perl-capture-tiny +Requires: perl, mbuffer, lzop, pv, perl-Config-IniFiles, perl-Capture-Tiny %if 0%{?_with_systemd} Requires: systemd >= 212 From fd71e794ad4e56afa54784516fd2c2e709d1723f Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Thu, 16 May 2019 11:38:39 -0500 Subject: [PATCH 02/42] Gracefully handle error when source dataset disappeared If the source dataset dissappeared before we were able to get the sanoid:sync property, do not set an error exit code. Handle this gracefully as this should not be considered an error. Fixes #380 --- syncoid | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 21e9d1e..98403ec 100755 --- a/syncoid +++ b/syncoid @@ -290,7 +290,10 @@ sub syncdataset { if (!defined $sync) { # zfs already printed the corresponding error - if ($exitcode < 2) { $exitcode = 2; } + if (dataset_exists($sourcehost, $sourcefs, $sourceisroot) and $exitcode < 2) { + $exitcode = 2; + } + return 0; } @@ -1147,6 +1150,37 @@ sub getzfsvalue { return $value; } +sub dataset_exists { + my ($rhost, $fs, $isroot) = @_; + + my $fsescaped = escapeshellparam($fs); + + my $mysudocmd; + if ($isroot) { + $mysudocmd = ''; + } + else { + $mysudocmd = $sudocmd; + } + + my $command = "$rhost $mysudocmd $zfscmd get -H creation $fsescaped"; + + if ($debug) { + print "$command\n"; + } + + open my $fh, '-|', "$command 2>&1"; + my $result = <$fh>; + close $fh; + + if ($result =~ /dataset does not exist/) { + warn "$fsescaped does not exist...\n"; + return 0; + } + + return 1; +} + sub readablebytes { my $bytes = shift; my $disp; From 784efe2d85f05e4bb3ca8f060ec1cfefc902b48c Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Thu, 16 May 2019 11:57:08 -0500 Subject: [PATCH 03/42] Remove debug warning --- syncoid | 1 - 1 file changed, 1 deletion(-) diff --git a/syncoid b/syncoid index 98403ec..c443b84 100755 --- a/syncoid +++ b/syncoid @@ -1174,7 +1174,6 @@ sub dataset_exists { close $fh; if ($result =~ /dataset does not exist/) { - warn "$fsescaped does not exist...\n"; return 0; } From 3892d73594c1e9faf14d65f5d0008095a005d98d Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Wed, 29 May 2019 17:13:31 -0500 Subject: [PATCH 04/42] Capture output in getzfsvalue and handle datasets that disappeared Change getzfsvalue() so that if called in array context, returns the value and the error as a two element list. This allows the caller to recieve the error from the zfs command. If the dataset went away before fetching the sanoid:sync property, just issue a warning and skip it. --- syncoid | 55 ++++++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/syncoid b/syncoid index c443b84..f7a4b66 100755 --- a/syncoid +++ b/syncoid @@ -286,12 +286,18 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); + my ($sync, $error) = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); if (!defined $sync) { # zfs already printed the corresponding error - if (dataset_exists($sourcehost, $sourcefs, $sourceisroot) and $exitcode < 2) { - $exitcode = 2; + if ($error =~ /\bdataset does not exist\b/) { + if (!$quiet) { print "WARN Skipping dataset (dataset no longer exists): $sourcefs...\n"; } + return 0; + } + else { + # print the error out and set exit code + print "ERROR: $error\n"; + if ($exitcode < 2) { $exitcode = 2 } } return 0; @@ -1137,47 +1143,22 @@ sub getzfsvalue { my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fsescaped\n"; } - open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; - my $value = ; - close FH; - - if (!defined $value) { - return undef; - } + my ($value, $error, $exit) = capture { + system("$rhost $mysudocmd $zfscmd get -H $property $fsescaped"); + }; my @values = split(/\t/,$value); $value = $values[2]; - return $value; -} -sub dataset_exists { - my ($rhost, $fs, $isroot) = @_; + my $wantarray = wantarray || 0; - my $fsescaped = escapeshellparam($fs); - - my $mysudocmd; - if ($isroot) { - $mysudocmd = ''; - } - else { - $mysudocmd = $sudocmd; + # If we are in scalar context and there is an error, print it out. + # Otherwise we assume the caller will deal with it. + if (!$wantarray and $error) { + print "ERROR getzfsvalue $fs $property: $error\n"; } - my $command = "$rhost $mysudocmd $zfscmd get -H creation $fsescaped"; - - if ($debug) { - print "$command\n"; - } - - open my $fh, '-|', "$command 2>&1"; - my $result = <$fh>; - close $fh; - - if ($result =~ /dataset does not exist/) { - return 0; - } - - return 1; + return $wantarray ? ($value, $error) : $value; } sub readablebytes { From 22fe41cb113f892efe8e9654f49f2a44427e9579 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 6 Jun 2019 08:04:00 +0200 Subject: [PATCH 05/42] redirect zfs receive error to stdout and only capture stdout for parsing of error codes, this way stderr is left alone and pv can work interactively --- syncoid | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/syncoid b/syncoid index 21e9d1e..e41a940 100755 --- a/syncoid +++ b/syncoid @@ -272,7 +272,6 @@ sub syncdataset { my ($sourcehost, $sourcefs, $targethost, $targetfs, $origin, $skipsnapshot) = @_; my $stdout; - my $stderr; my $exit; my $sourcefsescaped = escapeshellparam($sourcefs); @@ -518,7 +517,7 @@ sub syncdataset { if (defined($receivetoken)) { $sendoptions = getoptionsline(\@sendoptions, ('P','e','v','w')); my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -t $receivetoken"; - my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped 2>&1"; my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -527,13 +526,13 @@ sub syncdataset { if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - ($stdout, $stderr, $exit) = tee { + ($stdout, $exit) = tee_stdout { system("$synccmd") }; $exit == 0 or do { - if ($stderr =~ /\Qused in the initial send no longer exists\E/) { - if (!$quiet) { print "WARN: resetting partially receive state\n"; } + if ($stdout =~ /\Qused in the initial send no longer exists\E/) { + if (!$quiet) { print "WARN: resetting partially receive state because the snapshot source no longer exists\n"; } resetreceivestate($targethost,$targetfs,$targetisroot); # do an normal sync cycle return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, $origin); @@ -683,18 +682,18 @@ sub syncdataset { if ($nextsnapshot) { my $nextsnapshotescaped = escapeshellparam($nextsnapshot); my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped 2>&1"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - ($stdout, $stderr, $exit) = tee { + ($stdout, $exit) = tee_stdout { system("$synccmd") }; $exit == 0 or do { - if (!$resume && $stderr =~ /\Qcontains partially-complete state\E/) { + if (!$resume && $stdout =~ /\Qcontains partially-complete state\E/) { if (!$quiet) { print "WARN: resetting partially receive state\n"; } resetreceivestate($targethost,$targetfs,$targetisroot); system("$synccmd") == 0 or do { @@ -713,18 +712,18 @@ sub syncdataset { $matchingsnapescaped = escapeshellparam($matchingsnap); } else { my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped 2>&1"; my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - ($stdout, $stderr, $exit) = tee { + ($stdout, $exit) = tee_stdout { system("$synccmd") }; $exit == 0 or do { - if (!$resume && $stderr =~ /\Qcontains partially-complete state\E/) { + if (!$resume && $stdout =~ /\Qcontains partially-complete state\E/) { if (!$quiet) { print "WARN: resetting partially receive state\n"; } resetreceivestate($targethost,$targetfs,$targetisroot); system("$synccmd") == 0 or do { @@ -751,7 +750,7 @@ sub syncdataset { $sendoptions = getoptionsline(\@sendoptions, ('D','L','P','R','c','e','h','p','v','w')); my $sendcmd = "$sourcesudocmd $zfscmd send $sendoptions $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $receiveextraargs $forcedrecv $targetfsescaped 2>&1"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -760,12 +759,12 @@ sub syncdataset { if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - ($stdout, $stderr, $exit) = tee { + ($stdout, $exit) = tee_stdout { system("$synccmd") }; $exit == 0 or do { - if (!$resume && $stderr =~ /\Qcontains partially-complete state\E/) { + if (!$resume && $stdout =~ /\Qcontains partially-complete state\E/) { if (!$quiet) { print "WARN: resetting partially receive state\n"; } resetreceivestate($targethost,$targetfs,$targetisroot); system("$synccmd") == 0 or do { From 56f97b5bd28e6ebc372bfe61a10ca4572c6ed823 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 6 Jun 2019 08:09:42 +0200 Subject: [PATCH 06/42] capture the stderr of zfs send command in the case of a resumed replication where the send size can't be estimated (in which case pv is not working anyway, so we can break the interactivitiy) --- syncoid | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index e41a940..29b7319 100755 --- a/syncoid +++ b/syncoid @@ -526,9 +526,18 @@ sub syncdataset { if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - ($stdout, $exit) = tee_stdout { - system("$synccmd") - }; + if ($pvsize == 0) { + # we need to capture the error of zfs send, this will render pv useless but in this case + # it doesn't matter because we don't know the estimated send size (probably because + # the initial snapshot used for resumed send doesn't exist anymore) + ($stdout, $exit) = tee_stderr { + system("$synccmd") + }; + } else { + ($stdout, $exit) = tee_stdout { + system("$synccmd") + }; + } $exit == 0 or do { if ($stdout =~ /\Qused in the initial send no longer exists\E/) { From 0356af9d81684ace1daff505fa2a92e840eeec28 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 13 Jun 2019 14:31:05 +0200 Subject: [PATCH 07/42] Revert "Changed (and simplified) DST handling (see here: https://github.com/jimsalterjrs/sanoid/issues/155)" This reverts commit cf496dcf86bbdc5b90d6f4ddaa8d46db53c76710. --- README.md | 2 +- sanoid | 52 +++++++++++++++++++++++++++++++++---- tests/2_dst_handling/run.sh | 4 +-- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5ce4e8a..bc8ed83 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ More prosaically, you can use Sanoid to create, automatically thin, and monitor * * * * * TZ=UTC /usr/local/bin/sanoid --cron ``` -**`IMPORTANT NOTE`**: using a local timezone will result in a single hourly snapshot to be **skipped** during `daylight->nodaylight` transition. To avoid that, using UTC as timezone is recommend whenever possible. +`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times And its /etc/sanoid/sanoid.conf might look something like this: diff --git a/sanoid b/sanoid index c28a9de..e54800f 100755 --- a/sanoid +++ b/sanoid @@ -357,6 +357,19 @@ sub take_snapshots { my @newsnaps; + # get utc timestamp of the current day for DST check + my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'}); + my ($isdst) = (localtime($daystartUtc))[8]; + my $dstOffset = 0; + + if ($isdst ne $datestamp{'isdst'}) { + # current dst is different then at the beginning og the day + if ($isdst) { + # DST ended in the current day + $dstOffset = 60*60; + } + } + if ($args{'verbose'}) { print "INFO: taking snapshots...\n"; } foreach my $section (keys %config) { if ($section =~ /^template/) { next; } @@ -380,6 +393,9 @@ sub take_snapshots { my @preferredtime; my $lastpreferred; + # to avoid duplicates with DST + my $dateSuffix = ""; + if ($type eq 'frequently') { my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'}); @@ -399,6 +415,13 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); + + if ($dstOffset ne 0) { + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + # DST ended, avoid duplicates + $dateSuffix = "_y"; + } if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's } elsif ($type eq 'daily') { push @preferredtime,0; # try to hit 0 seconds @@ -408,10 +431,29 @@ sub take_snapshots { push @preferredtime,($datestamp{'mon'}-1); # january is month 0 push @preferredtime,$datestamp{'year'}; $lastpreferred = timelocal(@preferredtime); - if ($lastpreferred > time()) { - $preferredtime[3] -= 1; # preferred time is later today - so look at yesterday's - $lastpreferred = timelocal(@preferredtime); + + # timelocal doesn't take DST into account + $lastpreferred += $dstOffset; + + # check if the planned time has different DST flag than the current + my ($isdst) = (localtime($lastpreferred))[8]; + if ($isdst ne $datestamp{'isdst'}) { + if (!$isdst) { + # correct DST difference + $lastpreferred -= 60*60; + } } + + if ($lastpreferred > time()) { + $lastpreferred -= 60*60*24; + + if ($dstOffset ne 0) { + # because we are going back one day + # the DST difference has to be accounted + # for in reverse now + $lastpreferred -= 2*$dstOffset; + } + } # preferred time is later today - so look at yesterday's } elsif ($type eq 'weekly') { # calculate offset in seconds for the desired weekday my $offset = 0; @@ -463,9 +505,9 @@ sub take_snapshots { # use zfs (atomic) recursion if specified in config if ($config{$section}{'zfs_recursion'}) { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type\@"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type\@"); } else { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); } } } diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh index e86ca6e..7d7774e 100755 --- a/tests/2_dst_handling/run.sh +++ b/tests/2_dst_handling/run.sh @@ -13,7 +13,7 @@ set -x POOL_NAME="sanoid-test-2" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="0a6336ccdc948c69563cb56994d190aebbc9b21588aef17bb97e51ae074f879a" +RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c" # UTC timestamp of start and end START="1509141600" @@ -49,6 +49,6 @@ done saveSnapshotList "${POOL_NAME}" "${RESULT}" # hourly daily monthly -verifySnapshotList "${RESULT}" 72 3 1 "${RESULT_CHECKSUM}" +verifySnapshotList "${RESULT}" 73 3 1 "${RESULT_CHECKSUM}" # one more hour because of DST From cb94fa1d16380c8ad48e3112c144a59db9240515 Mon Sep 17 00:00:00 2001 From: George Hartzell Date: Thu, 13 Jun 2019 14:34:08 -0700 Subject: [PATCH 08/42] Documentation buglet, --sshkey specifies the *key* The `--sshkey` option specifies the ssh key, *not* the ssh public key. Empirically, it fails if you point it at a public key and works if you point it at the private part. Make sense since it's being used as the arg to `-i` in the `ssh` command line. --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 29b7319..551bf36 100755 --- a/syncoid +++ b/syncoid @@ -1799,7 +1799,7 @@ Options: --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filterd as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ... --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filterd as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ... - --sshkey=FILE Specifies a ssh public key to use to connect + --sshkey=FILE Specifies a ssh key to use to connect --sshport=PORT Connects to remote on a particular port --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set --sshoption|o=OPTION Passes OPTION to ssh for remote usage. Can be specified multiple times From 82230f248c5220da903e97df4183e2ac41ab0459 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 14 Jun 2019 07:53:40 +0200 Subject: [PATCH 09/42] improved dst handling to only suffix the duplicate hourly snapshot on DST change --- sanoid | 71 ++++++++++++++++++++++++++++++------- tests/1_one_year/run.sh | 2 +- tests/2_dst_handling/run.sh | 2 +- tests/common/lib.sh | 4 +-- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/sanoid b/sanoid index e54800f..224e91a 100755 --- a/sanoid +++ b/sanoid @@ -15,6 +15,7 @@ use File::Path; # for rmtree command in use_prune use Getopt::Long qw(:config auto_version auto_help); use Pod::Usage; # pod2usage use Time::Local; # to parse dates in reverse +use Capture::Tiny ':all'; my %args = ("configdir" => "/etc/sanoid"); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", @@ -394,7 +395,7 @@ sub take_snapshots { my $lastpreferred; # to avoid duplicates with DST - my $dateSuffix = ""; + my $handleDst = 0; if ($type eq 'frequently') { my $frequentslice = int($datestamp{'min'} / $config{$section}{'frequent_period'}); @@ -420,7 +421,7 @@ sub take_snapshots { # timelocal doesn't take DST into account $lastpreferred += $dstOffset; # DST ended, avoid duplicates - $dateSuffix = "_y"; + $handleDst = 1; } if ($lastpreferred > time()) { $lastpreferred -= 60*60; } # preferred time is later this hour - so look at last hour's } elsif ($type eq 'daily') { @@ -503,11 +504,19 @@ sub take_snapshots { %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; + my $flags = ""; # use zfs (atomic) recursion if specified in config if ($config{$section}{'zfs_recursion'}) { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type\@"); + $flags .= "r"; + } + if ($handleDst) { + $flags .= "d"; + } + + if ($flags ne "") { + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type\@$flags"); } else { - push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}${dateSuffix}_$type"); + push(@newsnaps, "$path\@autosnap_$datestamp{'sortable'}_$type"); } } } @@ -519,9 +528,18 @@ sub take_snapshots { my $extraMessage = ""; my @split = split '@', $snap, -1; my $recursiveFlag = 0; + my $dstHandling = 0; if (scalar(@split) == 3) { - $recursiveFlag = 1; - $extraMessage = " (zfs recursive)"; + my $flags = $split[2]; + if (index($flags, "r") != -1) { + $recursiveFlag = 1; + $extraMessage = " (zfs recursive)"; + chop $snap; + } + if (index($flags, "d") != -1) { + $dstHandling = 1; + chop $snap; + } chop $snap; } my $dataset = $split[0]; @@ -548,13 +566,40 @@ sub take_snapshots { } if ($args{'verbose'}) { print "taking snapshot $snap$extraMessage\n"; } if (!$args{'readonly'}) { - if ($recursiveFlag) { - system($zfs, "snapshot", "-r", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; - } else { - system($zfs, "snapshot", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; - } + my $stderr; + my $exit; + ($stderr, $exit) = tee_stderr { + if ($recursiveFlag) { + system($zfs, "snapshot", "-r", "$snap"); + } else { + system($zfs, "snapshot", "$snap"); + } + }; + + $exit == 0 or do { + if ($dstHandling) { + if ($stderr =~ /already exists/) { + $exit = 0; + $snap =~ s/_([a-z]+)$/dst_$1/g; + if ($args{'verbose'}) { print "taking dst snapshot $snap$extraMessage\n"; } + if ($recursiveFlag) { + system($zfs, "snapshot", "-r", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; + } else { + system($zfs, "snapshot", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + } + } + } + }; + + $exit == 0 or do { + if ($recursiveFlag) { + warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; + } else { + warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + } + }; } if ($config{$dataset}{'post_snapshot_script'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { diff --git a/tests/1_one_year/run.sh b/tests/1_one_year/run.sh index 88100da..fe76946 100755 --- a/tests/1_one_year/run.sh +++ b/tests/1_one_year/run.sh @@ -10,7 +10,7 @@ set -x POOL_NAME="sanoid-test-1" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="68c67161a59d0e248094a66061972f53613067c9db52ad981030f36bc081fed7" +RESULT_CHECKSUM="92f2c7afba94b59e8a6f6681705f0aa3f1c61e4aededaa38281e0b7653856935" # UTC timestamp of start and end START="1483225200" diff --git a/tests/2_dst_handling/run.sh b/tests/2_dst_handling/run.sh index 7d7774e..3231631 100755 --- a/tests/2_dst_handling/run.sh +++ b/tests/2_dst_handling/run.sh @@ -13,7 +13,7 @@ set -x POOL_NAME="sanoid-test-2" POOL_TARGET="" # root RESULT="/tmp/sanoid_test_result" -RESULT_CHECKSUM="a916d9cd46f4b80f285d069f3497d02671bbb1bfd12b43ef93531cbdaf89d55c" +RESULT_CHECKSUM="846372ef238f2182b382c77a73ecddf99aa82f28cc9995bcc95592cc78305463" # UTC timestamp of start and end START="1509141600" diff --git a/tests/common/lib.sh b/tests/common/lib.sh index b070da2..904c98f 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -61,9 +61,9 @@ function saveSnapshotList { # clear the seconds for comparing if [ "$unamestr" == 'FreeBSD' ]; then - sed -i '' 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" + sed -i '' 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]/\100/g' "${RESULT}" else - sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]_/\100_/g' "${RESULT}" + sed -i 's/\(autosnap_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9]:[0-9][0-9]:\)[0-9][0-9]/\100/g' "${RESULT}" fi } From 979109481684b4dfd123131a5abb7ffb9d3ac744 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 14 Jun 2019 15:45:44 +0200 Subject: [PATCH 10/42] some FreeBSD related fixes --- syncoid | 3 ++- tests/syncoid/5_reset_resume_state/run.sh | 21 +++++++++------------ tests/syncoid/6_reset_resume_state2/run.sh | 21 +++++++++------------ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/syncoid b/syncoid index 29b7319..f842d4d 100755 --- a/syncoid +++ b/syncoid @@ -773,7 +773,8 @@ sub syncdataset { }; $exit == 0 or do { - if (!$resume && $stdout =~ /\Qcontains partially-complete state\E/) { + # FreeBSD reports "dataset is busy" instead of "contains partially-complete state" + if (!$resume && ($stdout =~ /\Qcontains partially-complete state\E/ || $stdout =~ /\Qdataset is busy\E/)) { if (!$quiet) { print "WARN: resetting partially receive state\n"; } resetreceivestate($targethost,$targetfs,$targetisroot); system("$synccmd") == 0 or do { diff --git a/tests/syncoid/5_reset_resume_state/run.sh b/tests/syncoid/5_reset_resume_state/run.sh index 6e1b22b..6e71002 100755 --- a/tests/syncoid/5_reset_resume_state/run.sh +++ b/tests/syncoid/5_reset_resume_state/run.sh @@ -23,7 +23,7 @@ function cleanUp { # export pool in any case trap cleanUp EXIT -zfs create "${POOL_NAME}"/src -o mountpoint="${MOUNT_TARGET}" +zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200 @@ -31,19 +31,16 @@ dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200 ../../../syncoid --debug --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst & syncoid_pid=$! sleep 5 -list_descendants () -{ - local children=$(ps -o pid= --ppid "$1") - - for pid in $children - do - list_descendants "$pid" - done - - echo "$children" +function getcpid() { + cpids=$(pgrep -P "$1"|xargs) + for cpid in $cpids; + do + echo "$cpid" + getcpid "$cpid" + done } -kill $(list_descendants $$) || true +kill $(getcpid $$) || true wait sleep 1 diff --git a/tests/syncoid/6_reset_resume_state2/run.sh b/tests/syncoid/6_reset_resume_state2/run.sh index e227223..1afc921 100755 --- a/tests/syncoid/6_reset_resume_state2/run.sh +++ b/tests/syncoid/6_reset_resume_state2/run.sh @@ -23,7 +23,7 @@ function cleanUp { # export pool in any case trap cleanUp EXIT -zfs create "${POOL_NAME}"/src -o mountpoint="${MOUNT_TARGET}" +zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src ../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200 @@ -32,19 +32,16 @@ zfs snapshot "${POOL_NAME}"/src@big ../../../syncoid --debug --no-sync-snap --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst & syncoid_pid=$! sleep 5 -list_descendants () -{ - local children=$(ps -o pid= --ppid "$1") - - for pid in $children - do - list_descendants "$pid" - done - - echo "$children" +function getcpid() { + cpids=$(pgrep -P "$1"|xargs) + for cpid in $cpids; + do + echo "$cpid" + getcpid "$cpid" + done } -kill $(list_descendants $$) || true +kill $(getcpid $$) || true wait sleep 1 From dee9d1817fa113f5cb68fbec6d64d32818475720 Mon Sep 17 00:00:00 2001 From: "Gabriel A. Devenyi" Date: Fri, 14 Jun 2019 11:50:34 -0400 Subject: [PATCH 11/42] Add ability to configure pv --- syncoid | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/syncoid b/syncoid index 29b7319..326f614 100755 --- a/syncoid +++ b/syncoid @@ -16,6 +16,7 @@ use Sys::Hostname; use Capture::Tiny ':all'; my $mbuffer_size = "16M"; +my $pvoptions = "-p -t -e -r -b"; # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? @@ -24,7 +25,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback", - "create-bookmark", + "create-bookmark", "pv-options=s" => \$pvoptions, "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -1225,13 +1226,13 @@ sub buildsynccmd { } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $bwlimit $mbufferoptions |"; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd $pvoptions -s $pvsize |"; } $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. #$synccmd = "$sendcmd | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; - if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd $pvoptions -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost "; @@ -1254,7 +1255,7 @@ sub buildsynccmd { $synccmd .= " | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. @@ -1268,7 +1269,7 @@ sub buildsynccmd { $synccmd .= " | "; if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$compressargs{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost "; @@ -1791,6 +1792,7 @@ Options: --source-bwlimit= Bandwidth limit in bytes/kbytes/etc per second on the source transfer --target-bwlimit= Bandwidth limit in bytes/kbytes/etc per second on the target transfer --mbuffer-size=VALUE Specify the mbuffer size (default: 16M), please refer to mbuffer(1) manual page. + --pv-options=OPTIONS Configure how pv displays the progress bar, default '-p -t -e -r -b' --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) From a85ad93a18c6a8873a7dd8b2c0bd36d66c5d9031 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 18 Jun 2019 08:32:11 +0200 Subject: [PATCH 12/42] added needed verbose flag for send size estimation (at least for latest FreeBSD codebase) --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 975d7c5..ad62930 100755 --- a/syncoid +++ b/syncoid @@ -1602,9 +1602,9 @@ sub getsendsize { if (defined($receivetoken)) { $sendoptions = getoptionsline(\@sendoptions, ('e')); } else { - $sendoptions = getoptionsline(\@sendoptions, ('D','L','R','c','e','h','p','v','w')); + $sendoptions = getoptionsline(\@sendoptions, ('D','L','R','c','e','h','p','w')); } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nP $snaps"; + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nvP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } open FH, "$getsendsizecmd 2>&1 |"; From 056193ec040fad2ceb48f579d8f8f65336d38150 Mon Sep 17 00:00:00 2001 From: Donald Webster Date: Fri, 20 Sep 2019 13:28:59 -0700 Subject: [PATCH 13/42] Update to v2.0.2. --- CHANGELIST | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- sanoid | 2 +- 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 0fc90be..a151075 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,84 @@ +2.0.2 + 9791094 some FreeBSD related fixes + 56f97b5 capture the stderr of zfs send command in the case of a resumed replication where the send size can't be estimated (in which case pv is not working anyway, so we can break the interactivitiy) + 22fe41c redirect zfs receive error to stdout and only capture stdout for parsing of error codes, this way stderr is left alone and pv can work interactively + 5d7ee40 fix centos package name + 83c5e2c added new dependency to all packages and install instructions + 0537de5 test case for reseting the resume state if it's invalid + 6eb1be3 handle resume states where the source snapshot was already deleted + 16fceba moved common used variables to top of function + 0ef2bd2 added test to verify the last changes to handle partially received replication streams + 1046f7f use capture::tiny for teeing commands + eef598c Update rules + 67ff4fe Update INSTALL.md + 522bdec fixed ordering of snapshots with the same creation timestamp + ce93d89 Declare that sanoid service 'Wants' sanoid-prune + +2.0.1 + ea90108 revert -r in zfs get (not needed here) + 8d8df7e Add -r to zfs get -t snap for zfs 8.1 + 607aad0 Add -r to zfs get -Hpt snap for zfs 8.1 + faf268b Added line to enable sanoid-prune.service + 1900eca added verbose cmd line flags for journald logging + 90b8c92 sanoid-prune.service is indeed referenced and run after the sanoid.service + 17e15c2 Fix systemd service definitions. + 68cc59a Update INSTALL.md + 56d7c59 clarify --bwlimit units + 681aa61 clarify bandwidth limit units + ffa6ad3 Fix gentoo package snafu - need a sanoid directory to store the files + df1e344 Fix gentoo package snafu - need a sanoid directory to store the files + cf496dc Changed (and simplified) DST handling (see here: https://github.com/jimsalterjrs/sanoid/issues/155) + 736d031 initial replication without intermediate snapshots should use the created sync snapshot + e0e7b0d added bwlimit units + a97da7d Added bwlimit units + e014fd7 check if ssh connection works + eefc659 update the install instruction with recent changes + 25c5e2e allow new zfs send/recv hold parameter (introduced in recent master) and drop invalid raw parameter for send size estimation for resumeable send + c543f9a use ordinary replication if clone recreation fails + d3f19c8 Sanoid ebuild for Gentoo + a748b41 if bookmark creation fails use guid bases suffix based fallback name + 99b439e return if there is nothing to do + 5812b57 check for invalid send/receive option syntax + 10b2012 removed debug output + 7d522fa fix unitialized variable + 1053e93 missed sudo command for bookmark creation if source host is remote + f0fec53 fix root pool parsing for remote hosts + c5eebbe implemented parsing of provided zfs send/recv options and whitelisting for each use case as needed + 5107380 codestyle + d68d7b4 implemented creation of zfs bookmarks + 7141eab fix bookmark edge case where replication was already done to the latest snapshot (+ test) + ea9e989 only list filesystems and volumes + 132c765 error out if no datasets are processed with recursive arg + 85561dd handle the case of mistyped dataset name nicely + c12b6d7 check pool capability for resumeable replication instead of checking for support in the zfs module + 7ef435c add "hotspare" template (for local, hourly replication targets) + c3e20fd FreeBSD sed needs a different syntax + ea482ce make tests work on FreeBSD + ab12540 added message if zfs recursion is used and fixed ordinary recursion in combination with zfs recursion + 0fc16b4 fixed uninitialzed variable warning and clarified recursive value + 271ede7 Revert "Revert "Zfs Recursion"" + e4e9065 removed debug output from excludes loop + 50a2374 Revert "Zfs Recursion" + 165faee remove spurious line break from HTML + f941d7f fix for #316 - CRITICAL ERROR: bookmarks couldn't be listed for + bd4eb49 Added in --sendoptions=OPTIONS and --recvoptions=OPTIONS to inject OPTIONS enabling things like syncoid --sendoptions=-Lcep and syncoid --recvoptions="-x property" + ca76f42 syncoid: add '--mbuffer-size' option + c35c953 remove redundant loop + f8e0e00 preserve taking snapshots order always from yearly to frequently + 7cf64be pre/post snapshot script calls should be also dumped in --readonly mode + 4daafca prevent problems with shell expansion and codestyle + 1605a60 CLIMATE-1151: added config option to use zfs recursion + 8d88c47 improved documentation on --no-command-checks + ac80a75 made time units case insensitive and removed monthlies + cfab4ea added/fixed documentation + 89f1d4e run compress commands bare too + 941a770 updated comment regarding command existence check + be50481 use more portable/POSIX compatible ways to check for command existence + 9b2c2f2 don't use hardcoded paths + abf3d45 check for partial resume state for non resume style transfers, if so reset state and retry + 0f669a3 Cleaned up Configuration section to match the rest of the document + 429da3a Add comprehensive installation instructions + 2.0.0 [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) [syncoid] added force delete flag (@phreaker0) [sanoid] removed sleeping between snapshot taking (@phreaker0) diff --git a/VERSION b/VERSION index 38f77a6..e9307ca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1 +2.0.2 diff --git a/sanoid b/sanoid index c28a9de..5a842bb 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.1'; +$::VERSION = '2.0.2'; my $MINIMUM_DEFAULTS_VERSION = 2; use strict; From b04bd2f02d2ea3762e55f215d1367c0eb7a22a85 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 25 Sep 2019 22:22:00 +0200 Subject: [PATCH 14/42] proper changelog for the last two version and packaging updates --- CHANGELIST | 104 +++++------------- packages/debian/changelog | 35 ++++++ packages/gentoo/sys-fs/sanoid/Manifest | 4 +- ...anoid-2.0.1.ebuild => sanoid-2.0.2.ebuild} | 0 packages/rhel/sanoid.spec | 4 +- syncoid | 2 +- 6 files changed, 66 insertions(+), 83 deletions(-) rename packages/gentoo/sys-fs/sanoid/{sanoid-2.0.1.ebuild => sanoid-2.0.2.ebuild} (100%) diff --git a/CHANGELIST b/CHANGELIST index a151075..e66940d 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,83 +1,29 @@ -2.0.2 - 9791094 some FreeBSD related fixes - 56f97b5 capture the stderr of zfs send command in the case of a resumed replication where the send size can't be estimated (in which case pv is not working anyway, so we can break the interactivitiy) - 22fe41c redirect zfs receive error to stdout and only capture stdout for parsing of error codes, this way stderr is left alone and pv can work interactively - 5d7ee40 fix centos package name - 83c5e2c added new dependency to all packages and install instructions - 0537de5 test case for reseting the resume state if it's invalid - 6eb1be3 handle resume states where the source snapshot was already deleted - 16fceba moved common used variables to top of function - 0ef2bd2 added test to verify the last changes to handle partially received replication streams - 1046f7f use capture::tiny for teeing commands - eef598c Update rules - 67ff4fe Update INSTALL.md - 522bdec fixed ordering of snapshots with the same creation timestamp - ce93d89 Declare that sanoid service 'Wants' sanoid-prune +2.0.2 [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) + [syncoid] changed and simplified DST handling (@shodanshok) + [syncoid] reset partially resume state automatically (@phreaker0) + [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) + [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) + [syncoid] don't use hardcoded paths (@phreaker0) + [syncoid] fix for special setup with listsnapshots=on (@phreaker0) + [syncoid] check ssh connection on startup (@phreaker0) + [syncoid] fix edge case with initial send and no-stream option (@phreaker0) + [syncoid] fallback to normal replication if clone recreation fails (@phreaker0) + [packaging] ebuild for gentoo (@thehaven) + [syncoid] support for zfs bookmark creation (@phreaker0) + [syncoid] fixed bookmark edge cases (@phreaker0) + [syncoid] handle invalid dataset paths nicely (@phreaker0) + [syncoid] fixed resume support check to be zpool based (@phreaker0) + [sanoid] added hotspare template (@jimsalterjrs) + [syncoid] support for advanced zfs send/recv options (@clinta, @phreaker0) + [syncoid] option to change mbuffer size (@TerraTech) + [tests] fixes for FreeBSD (@phreaker0) + [sanoid] support for zfs recursion (@jMichaelA, @phreaker0) + [syncoid] fixed bookmark handling for volumens (@ppcontrib) + [sanoid] allow time units for monitoring warn/crit values (@phreaker0) -2.0.1 - ea90108 revert -r in zfs get (not needed here) - 8d8df7e Add -r to zfs get -t snap for zfs 8.1 - 607aad0 Add -r to zfs get -Hpt snap for zfs 8.1 - faf268b Added line to enable sanoid-prune.service - 1900eca added verbose cmd line flags for journald logging - 90b8c92 sanoid-prune.service is indeed referenced and run after the sanoid.service - 17e15c2 Fix systemd service definitions. - 68cc59a Update INSTALL.md - 56d7c59 clarify --bwlimit units - 681aa61 clarify bandwidth limit units - ffa6ad3 Fix gentoo package snafu - need a sanoid directory to store the files - df1e344 Fix gentoo package snafu - need a sanoid directory to store the files - cf496dc Changed (and simplified) DST handling (see here: https://github.com/jimsalterjrs/sanoid/issues/155) - 736d031 initial replication without intermediate snapshots should use the created sync snapshot - e0e7b0d added bwlimit units - a97da7d Added bwlimit units - e014fd7 check if ssh connection works - eefc659 update the install instruction with recent changes - 25c5e2e allow new zfs send/recv hold parameter (introduced in recent master) and drop invalid raw parameter for send size estimation for resumeable send - c543f9a use ordinary replication if clone recreation fails - d3f19c8 Sanoid ebuild for Gentoo - a748b41 if bookmark creation fails use guid bases suffix based fallback name - 99b439e return if there is nothing to do - 5812b57 check for invalid send/receive option syntax - 10b2012 removed debug output - 7d522fa fix unitialized variable - 1053e93 missed sudo command for bookmark creation if source host is remote - f0fec53 fix root pool parsing for remote hosts - c5eebbe implemented parsing of provided zfs send/recv options and whitelisting for each use case as needed - 5107380 codestyle - d68d7b4 implemented creation of zfs bookmarks - 7141eab fix bookmark edge case where replication was already done to the latest snapshot (+ test) - ea9e989 only list filesystems and volumes - 132c765 error out if no datasets are processed with recursive arg - 85561dd handle the case of mistyped dataset name nicely - c12b6d7 check pool capability for resumeable replication instead of checking for support in the zfs module - 7ef435c add "hotspare" template (for local, hourly replication targets) - c3e20fd FreeBSD sed needs a different syntax - ea482ce make tests work on FreeBSD - ab12540 added message if zfs recursion is used and fixed ordinary recursion in combination with zfs recursion - 0fc16b4 fixed uninitialzed variable warning and clarified recursive value - 271ede7 Revert "Revert "Zfs Recursion"" - e4e9065 removed debug output from excludes loop - 50a2374 Revert "Zfs Recursion" - 165faee remove spurious line break from HTML - f941d7f fix for #316 - CRITICAL ERROR: bookmarks couldn't be listed for - bd4eb49 Added in --sendoptions=OPTIONS and --recvoptions=OPTIONS to inject OPTIONS enabling things like syncoid --sendoptions=-Lcep and syncoid --recvoptions="-x property" - ca76f42 syncoid: add '--mbuffer-size' option - c35c953 remove redundant loop - f8e0e00 preserve taking snapshots order always from yearly to frequently - 7cf64be pre/post snapshot script calls should be also dumped in --readonly mode - 4daafca prevent problems with shell expansion and codestyle - 1605a60 CLIMATE-1151: added config option to use zfs recursion - 8d88c47 improved documentation on --no-command-checks - ac80a75 made time units case insensitive and removed monthlies - cfab4ea added/fixed documentation - 89f1d4e run compress commands bare too - 941a770 updated comment regarding command existence check - be50481 use more portable/POSIX compatible ways to check for command existence - 9b2c2f2 don't use hardcoded paths - abf3d45 check for partial resume state for non resume style transfers, if so reset state and retry - 0f669a3 Cleaned up Configuration section to match the rest of the document - 429da3a Add comprehensive installation instructions +2.0.1 [sanoid] fixed broken monthly warn/critical monitoring values in default template (@jimsalterjrs) + [sanoid] flag to force pruning while filesystem is in an active zfs send/recv (@shodanshok) + [syncoid] flags to disable rollbacks (@shodanshok) 2.0.0 [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) [syncoid] added force delete flag (@phreaker0) diff --git a/packages/debian/changelog b/packages/debian/changelog index 3722906..829e530 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,38 @@ +sanoid (2.0.2) unstable; urgency=medium + + [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) + [syncoid] changed and simplified DST handling (@shodanshok) + [syncoid] reset partially resume state automatically (@phreaker0) + [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) + [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) + [syncoid] don't use hardcoded paths (@phreaker0) + [syncoid] fix for special setup with listsnapshots=on (@phreaker0) + [syncoid] check ssh connection on startup (@phreaker0) + [syncoid] fix edge case with initial send and no-stream option (@phreaker0) + [syncoid] fallback to normal replication if clone recreation fails (@phreaker0) + [packaging] ebuild for gentoo (@thehaven) + [syncoid] support for zfs bookmark creation (@phreaker0) + [syncoid] fixed bookmark edge cases (@phreaker0) + [syncoid] handle invalid dataset paths nicely (@phreaker0) + [syncoid] fixed resume support check to be zpool based (@phreaker0) + [sanoid] added hotspare template (@jimsalterjrs) + [syncoid] support for advanced zfs send/recv options (@clinta, @phreaker0) + [syncoid] option to change mbuffer size (@TerraTech) + [tests] fixes for FreeBSD (@phreaker0) + [sanoid] support for zfs recursion (@jMichaelA, @phreaker0) + [syncoid] fixed bookmark handling for volumens (@ppcontrib) + [sanoid] allow time units for monitoring warn/crit values (@phreaker0) + + -- Jim Salter Fri, 20 Sep 2019 23:01:00 +0100 + +sanoid (2.0.1) unstable; urgency=medium + + [sanoid] fixed broken monthly warn/critical monitoring values in default template (@jimsalterjrs) + [sanoid] flag to force pruning while filesystem is in an active zfs send/recv (@shodanshok) + [syncoid] flags to disable rollbacks (@shodanshok) + + -- Jim Salter Fri, 14 Dec 2018 16:48:00 +0100 + sanoid (2.0.0) unstable; urgency=medium [overall] documentation updates, small fixes, more warnings (@sparky3387, @ljwobker, @phreaker0) diff --git a/packages/gentoo/sys-fs/sanoid/Manifest b/packages/gentoo/sys-fs/sanoid/Manifest index d18399f..d25575a 100644 --- a/packages/gentoo/sys-fs/sanoid/Manifest +++ b/packages/gentoo/sys-fs/sanoid/Manifest @@ -1,4 +1,4 @@ AUX sanoid.cron 45 BLAKE2B 3f6294bbbf485dc21a565cd2c8da05a42fb21cdaabdf872a21500f1a7338786c60d4a1fd188bbf81ce85f06a376db16998740996f47c049707a5109bdf02c052 SHA512 7676b32f21e517e8c84a097c7934b54097cf2122852098ea756093ece242125da3f6ca756a6fbb82fc348f84b94bfd61639e86e0bfa4bbe7abf94a8a4c551419 -DIST sanoid-2.0.1.tar.gz 106981 BLAKE2B 824b7271266ac9f9bf1fef5374a442215c20a4f139081f77d5d8db2ec7db9b8b349d9d0394c76f9d421a957853af64ff069097243f69e7e4b83a804f5ba992a6 SHA512 9d999b0f071bc3c3ca956df11e1501fd72a842f7d3315ede3ab3b5e0a36351100b6edbab8448bba65a2e187e4e8f77ff24671ed33b28f2fca9bb6ad0801aba9d -EBUILD sanoid-2.0.1.ebuild 796 BLAKE2B f3d633289d66c60fd26cb7731bc6b63533019f527aaec9ca8e5c0e748542d391153dbb55b17b8c981ca4fa4ae1fc8dc202b5480c13736fca250940b3b5ebb793 SHA512 d0143680c029ffe4ac37d97a979ed51527b4b8dd263d0c57e43a4650bf8a9bb8 +DIST sanoid-2.0.2.tar.gz 115797 BLAKE2B d00a038062df3dd8e77d3758c7b80ed6da0bac4931fb6df6adb72eeddb839c63d5129e0a281948a483d02165dad5a8505e1a55dc851360d3b366371038908142 SHA512 9d999b0f071bc3c3ca956df11e1501fd72a842f7d3315ede3ab3b5e0a36351100b6edbab8448bba65a2e187e4e8f77ff24671ed33b28f2fca9bb6ad0801aba9d +EBUILD sanoid-2.0.2.ebuild 796 BLAKE2B f3d633289d66c60fd26cb7731bc6b63533019f527aaec9ca8e5c0e748542d391153dbb55b17b8c981ca4fa4ae1fc8dc202b5480c13736fca250940b3b5ebb793 SHA512 d0143680c029ffe4ac37d97a979ed51527b4b8dd263d0c57e43a4650bf8a9bb8 EBUILD sanoid-9999.ebuild 776 BLAKE2B 416b8d04a9e5a84bce46d2a6f88eaefe03804944c03bc7f49b7a5b284b844212a6204402db3de3afa5d9c0545125d2631e7231c8cb2a3537bdcb10ea1be46b6a SHA512 98d8a30a13e75d7847ae9d60797d54078465bf75c6c6d9b6fd86075e342c0374 diff --git a/packages/gentoo/sys-fs/sanoid/sanoid-2.0.1.ebuild b/packages/gentoo/sys-fs/sanoid/sanoid-2.0.2.ebuild similarity index 100% rename from packages/gentoo/sys-fs/sanoid/sanoid-2.0.1.ebuild rename to packages/gentoo/sys-fs/sanoid/sanoid-2.0.2.ebuild diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 8ce0360..3aff0a9 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 2.0.0 +%global version 2.0.2 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Wed Sep 25 2019 Christoph Klaffl - 2.0.2 +- Bump to 2.0.2 * Wed Dec 04 2018 Christoph Klaffl - 2.0.0 - Bump to 2.0.0 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 diff --git a/syncoid b/syncoid index 975d7c5..e048391 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.0'; +$::VERSION = '2.0.2'; use strict; use warnings; From 96a48efd92a2949761349f480dc049624ad5bcd5 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 2 Oct 2019 16:42:38 +0200 Subject: [PATCH 15/42] prepare v2.0.3 bugfix release --- CHANGELIST | 4 +++- VERSION | 2 +- packages/debian/changelog | 6 ++++++ packages/rhel/sanoid.spec | 4 +++- sanoid | 2 +- syncoid | 2 +- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index e66940d..b83b717 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,5 +1,7 @@ +2.0.3 [sanoid] reverted DST handling and improved it as quickfix (@phreaker0) + 2.0.2 [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) - [syncoid] changed and simplified DST handling (@shodanshok) + [sanoid] changed and simplified DST handling (@shodanshok) [syncoid] reset partially resume state automatically (@phreaker0) [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) diff --git a/VERSION b/VERSION index e9307ca..50ffc5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.2 +2.0.3 diff --git a/packages/debian/changelog b/packages/debian/changelog index 829e530..7ea878b 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,9 @@ +sanoid (2.0.3) unstable; urgency=medium + + [sanoid] reverted DST handling and improved it as quickfix (@phreaker0) + + -- Jim Salter Wed, 02 Oct 2019 17:00:00 +0100 + sanoid (2.0.2) unstable; urgency=medium [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3aff0a9..ce912ea 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 2.0.2 +%global version 2.0.3 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Wed Oct 02 2019 Christoph Klaffl - 2.0.3 +- Bump to 2.0.3 * Wed Sep 25 2019 Christoph Klaffl - 2.0.2 - Bump to 2.0.2 * Wed Dec 04 2018 Christoph Klaffl - 2.0.0 diff --git a/sanoid b/sanoid index a17b91d..2b566ef 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.2'; +$::VERSION = '2.0.3'; my $MINIMUM_DEFAULTS_VERSION = 2; use strict; diff --git a/syncoid b/syncoid index e048391..f891099 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.2'; +$::VERSION = '2.0.3'; use strict; use warnings; From a1de743c3f02c009a6de2873735f07c1e26ce162 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 16 Oct 2019 09:08:17 +0200 Subject: [PATCH 16/42] fix sha512 checksum for gentoo packaging --- packages/gentoo/sys-fs/sanoid/Manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gentoo/sys-fs/sanoid/Manifest b/packages/gentoo/sys-fs/sanoid/Manifest index d25575a..06ef31c 100644 --- a/packages/gentoo/sys-fs/sanoid/Manifest +++ b/packages/gentoo/sys-fs/sanoid/Manifest @@ -1,4 +1,4 @@ AUX sanoid.cron 45 BLAKE2B 3f6294bbbf485dc21a565cd2c8da05a42fb21cdaabdf872a21500f1a7338786c60d4a1fd188bbf81ce85f06a376db16998740996f47c049707a5109bdf02c052 SHA512 7676b32f21e517e8c84a097c7934b54097cf2122852098ea756093ece242125da3f6ca756a6fbb82fc348f84b94bfd61639e86e0bfa4bbe7abf94a8a4c551419 -DIST sanoid-2.0.2.tar.gz 115797 BLAKE2B d00a038062df3dd8e77d3758c7b80ed6da0bac4931fb6df6adb72eeddb839c63d5129e0a281948a483d02165dad5a8505e1a55dc851360d3b366371038908142 SHA512 9d999b0f071bc3c3ca956df11e1501fd72a842f7d3315ede3ab3b5e0a36351100b6edbab8448bba65a2e187e4e8f77ff24671ed33b28f2fca9bb6ad0801aba9d +DIST sanoid-2.0.2.tar.gz 115797 BLAKE2B d00a038062df3dd8e77d3758c7b80ed6da0bac4931fb6df6adb72eeddb839c63d5129e0a281948a483d02165dad5a8505e1a55dc851360d3b366371038908142 SHA512 73e3d25dbdd58a78ffc4384584304e7230c5f31a660ce6d2a9b9d52a92a3796f1bc25ae865dbc74ce586cbd6169dbb038340f4a28e097e77ab3eb192b15773db EBUILD sanoid-2.0.2.ebuild 796 BLAKE2B f3d633289d66c60fd26cb7731bc6b63533019f527aaec9ca8e5c0e748542d391153dbb55b17b8c981ca4fa4ae1fc8dc202b5480c13736fca250940b3b5ebb793 SHA512 d0143680c029ffe4ac37d97a979ed51527b4b8dd263d0c57e43a4650bf8a9bb8 EBUILD sanoid-9999.ebuild 776 BLAKE2B 416b8d04a9e5a84bce46d2a6f88eaefe03804944c03bc7f49b7a5b284b844212a6204402db3de3afa5d9c0545125d2631e7231c8cb2a3537bdcb10ea1be46b6a SHA512 98d8a30a13e75d7847ae9d60797d54078465bf75c6c6d9b6fd86075e342c0374 From 6fed3f6c1bb501c23cec8ec74804422b4e9bbc6f Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 05:57:31 -0600 Subject: [PATCH 17/42] Add a debian/.gitignore --- packages/debian/.gitignore | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/debian/.gitignore diff --git a/packages/debian/.gitignore b/packages/debian/.gitignore new file mode 100644 index 0000000..7f9a468 --- /dev/null +++ b/packages/debian/.gitignore @@ -0,0 +1,7 @@ +*.debhelper +*.debhelper.log +*.substvars +debhelper-build-stamp +files +sanoid +tmp From 6a3e38b6437890a6a3fa9cdfa77614295d995a58 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 05:57:49 -0600 Subject: [PATCH 18/42] Set a Section --- packages/debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index a64dcde..f00a26f 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -1,5 +1,5 @@ Source: sanoid -Section: unknown +Section: utils Priority: optional Maintainer: Jim Salter Build-Depends: debhelper (>= 9) From 676c08c9f49300021477bc9e203485428b7451dc Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 05:58:19 -0600 Subject: [PATCH 19/42] Run wrap-and-sort -a --- packages/debian/control | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index f00a26f..b2b8e5c 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -10,5 +10,9 @@ Vcs-Browser: https://github.com/jimsalterjrs/sanoid Package: sanoid Architecture: all -Depends: ${misc:Depends}, ${perl:Depends}, zfsutils-linux | zfs, libconfig-inifiles-perl, libcapture-tiny-perl +Depends: libcapture-tiny-perl, + libconfig-inifiles-perl, + zfsutils-linux | zfs, + ${misc:Depends}, + ${perl:Depends} Description: Policy-driven snapshot management and replication tools From 52912a9bb8c48b5a52307f50164aa683579a08e5 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 05:59:09 -0600 Subject: [PATCH 20/42] Use install in debian/rules Using install is more ideomatic than mkdir + cp. --- packages/debian/rules | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/debian/rules b/packages/debian/rules index 8cadf58..7bd8622 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -9,14 +9,21 @@ DESTDIR = $(CURDIR)/debian/sanoid override_dh_auto_install: - @mkdir -p $(DESTDIR)/usr/sbin; \ - cp sanoid syncoid findoid sleepymutex $(DESTDIR)/usr/sbin; - @mkdir -p $(DESTDIR)/etc/sanoid; \ - cp sanoid.defaults.conf $(DESTDIR)/etc/sanoid; - @mkdir -p $(DESTDIR)/usr/share/doc/sanoid; \ - cp sanoid.conf $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example; - @mkdir -p $(DESTDIR)/lib/systemd/system; \ - cp debian/sanoid-prune.service debian/sanoid.timer $(DESTDIR)/lib/systemd/system; + install -d $(DESTDIR)/etc/sanoid + install -m 664 sanoid.defaults.conf $(DESTDIR)/etc/sanoid + + install -d $(DESTDIR)/lib/systemd/system + install -m 664 debian/sanoid-prune.service debian/sanoid.timer \ + $(DESTDIR)/lib/systemd/system + + install -d $(DESTDIR)/usr/sbin + install -m 775 \ + findoid sanoid sleepymutex syncoid \ + $(DESTDIR)/usr/sbin + + install -d $(DESTDIR)/usr/share/doc/sanoid + install -m 664 sanoid.conf \ + $(DESTDIR)/usr/share/doc/sanoid/sanoid.conf.example override_dh_installinit: dh_installinit --noscripts From c86301f4fd6d9ce8b6dedf6df868ec610a436c58 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 05:59:49 -0600 Subject: [PATCH 21/42] Wrap README.Debian --- packages/debian/sanoid.README.Debian | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/debian/sanoid.README.Debian b/packages/debian/sanoid.README.Debian index afaa36b..27fadfc 100644 --- a/packages/debian/sanoid.README.Debian +++ b/packages/debian/sanoid.README.Debian @@ -1 +1,2 @@ -To start, copy the example config file in /usr/share/doc/sanoid to /etc/sanoid/sanoid.conf. +To start, copy the example config file in /usr/share/doc/sanoid to +/etc/sanoid/sanoid.conf. From 2bbf4b50c3c75ac21af8ddb2c5c5eeb5ff48d3d7 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:00:05 -0600 Subject: [PATCH 22/42] Remove extraneous debian/copyright boilerplate --- packages/debian/copyright | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/debian/copyright b/packages/debian/copyright index c8ea7dc..3be2046 100644 --- a/packages/debian/copyright +++ b/packages/debian/copyright @@ -26,8 +26,3 @@ License: GPL-3.0+ . On Debian systems, the complete text of the GNU General Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". - -# Please also look if there are files or directories which have a -# different copyright/license attached and list them here. -# Please avoid picking licenses with terms that are more restrictive than the -# packaged work, as it may make Debian's contributions unacceptable upstream. From bcf1967cb4dc1835eb5cfd5aeb0c83cde724f2fc Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:00:15 -0600 Subject: [PATCH 23/42] Fix the syntax in debian/copyright's Source There is no need for angle brackets around URLs. --- packages/debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/copyright b/packages/debian/copyright index 3be2046..a43a0a9 100644 --- a/packages/debian/copyright +++ b/packages/debian/copyright @@ -1,6 +1,6 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: sanoid -Source: +Source: https://github.com/jimsalterjrs/sanoid Files: * Copyright: 2017 Jim Salter From 59e181e61d2e0d9f29834fc65b2201f75d1490a4 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:00:45 -0600 Subject: [PATCH 24/42] Depend on systemd For sanoid to function, it needs to be run periodically. Currently, the package only has a systemd timer. --- packages/debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/debian/control b/packages/debian/control index b2b8e5c..099cd30 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -12,6 +12,7 @@ Package: sanoid Architecture: all Depends: libcapture-tiny-perl, libconfig-inifiles-perl, + systemd, zfsutils-linux | zfs, ${misc:Depends}, ${perl:Depends} From 6c4477f7729f6aba09b296219a4b1676172594ac Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:05:28 -0600 Subject: [PATCH 25/42] Recommends: utilities for syncoid --- packages/debian/control | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/debian/control b/packages/debian/control index 099cd30..8d4dc3b 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -16,4 +16,9 @@ Depends: libcapture-tiny-perl, zfsutils-linux | zfs, ${misc:Depends}, ${perl:Depends} +Recommends: gzip, + lzop, + mbuffer, + openssh-client | ssh-client, + pv Description: Policy-driven snapshot management and replication tools From fd9d4bc54a40695eab140386fd2b4a533b39670f Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:06:00 -0600 Subject: [PATCH 26/42] Bump Standards-Version (no changes) --- packages/debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index 8d4dc3b..361435e 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -3,7 +3,7 @@ Section: utils Priority: optional Maintainer: Jim Salter Build-Depends: debhelper (>= 9) -Standards-Version: 3.9.8 +Standards-Version: 4.1.1 Homepage: https://github.com/jimsalterjrs/sanoid Vcs-Git: https://github.com/jimsalterjrs/sanoid.git Vcs-Browser: https://github.com/jimsalterjrs/sanoid From cac90382e2fc047a49d7a558cf64a2f0cec61bda Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:06:22 -0600 Subject: [PATCH 27/42] Use debhelper compat 10 --- packages/debian/compat | 2 +- packages/debian/control | 2 +- packages/debian/rules | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/debian/compat b/packages/debian/compat index ec63514..f599e28 100644 --- a/packages/debian/compat +++ b/packages/debian/compat @@ -1 +1 @@ -9 +10 diff --git a/packages/debian/control b/packages/debian/control index 361435e..8465243 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -2,7 +2,7 @@ Source: sanoid Section: utils Priority: optional Maintainer: Jim Salter -Build-Depends: debhelper (>= 9) +Build-Depends: debhelper (>= 10) Standards-Version: 4.1.1 Homepage: https://github.com/jimsalterjrs/sanoid Vcs-Git: https://github.com/jimsalterjrs/sanoid.git diff --git a/packages/debian/rules b/packages/debian/rules index 7bd8622..51e52af 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -5,7 +5,7 @@ #export DH_VERBOSE = 1 %: - dh $@ --with systemd + dh $@ DESTDIR = $(CURDIR)/debian/sanoid override_dh_auto_install: From af115dcbe200de7d6db6dd59e0f96f6bcc67a616 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:09:46 -0600 Subject: [PATCH 28/42] Add debian/TODO This has some notes on things that need to change before this package can be submitted to Debian. --- packages/debian/TODO | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 packages/debian/TODO diff --git a/packages/debian/TODO b/packages/debian/TODO new file mode 100644 index 0000000..1bc0fb4 --- /dev/null +++ b/packages/debian/TODO @@ -0,0 +1,12 @@ +- This package needs to be a 3.0 (quilt) format, not 3.0 (native). + - Fix the changelog + - Move the packaging out to a separate repository, or at a minimum, + a separate branch. +- Provide an extended description in debian/control +- Figure out a plan for sanoid.defaults.conf. It is not supposed to be + edited, so it shouldn't be installed in /etc. At a minimum, install + it under /usr and make a symlink, but preferably patch sanoid to look + there directly. +- Man pages are necessary for all the utilities installed. + - With these, there is probably no need to ship README.md. +- Break out syncoid into a separate package? From a7662511c8953cf86b53dbc655c35a0301ebc298 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Sun, 26 Nov 2017 06:14:16 -0600 Subject: [PATCH 29/42] Add myself to debian/copyright --- packages/debian/copyright | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/debian/copyright b/packages/debian/copyright index a43a0a9..75d3960 100644 --- a/packages/debian/copyright +++ b/packages/debian/copyright @@ -8,6 +8,7 @@ License: GPL-3.0+ Files: debian/* Copyright: 2017 Jim Salter + 2017 Richard Laager License: GPL-3.0+ License: GPL-3.0+ From 55e362c951905628d66fd3919f6798ca42535378 Mon Sep 17 00:00:00 2001 From: Richard Laager Date: Thu, 30 Nov 2017 20:20:42 -0600 Subject: [PATCH 30/42] Bump Standards-Version to 4.1.2 (no changes) --- packages/debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index 8465243..da70b65 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -3,7 +3,7 @@ Section: utils Priority: optional Maintainer: Jim Salter Build-Depends: debhelper (>= 10) -Standards-Version: 4.1.1 +Standards-Version: 4.1.2 Homepage: https://github.com/jimsalterjrs/sanoid Vcs-Git: https://github.com/jimsalterjrs/sanoid.git Vcs-Browser: https://github.com/jimsalterjrs/sanoid From 0e5c2e1cff7cfe5dcaf893f9c8fa838c3cf22b8a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:19:56 +0100 Subject: [PATCH 31/42] improve dataset detection by only including mounted datasets --- findoid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/findoid b/findoid index 48301a4..67b5de6 100755 --- a/findoid +++ b/findoid @@ -102,14 +102,19 @@ sub getdataset { my ($path) = @_; - open FH, "$zfs list -Ho mountpoint |"; + open FH, "$zfs list -H -t filesystem -o mountpoint,mounted |"; my @datasets = ; close FH; my @matchingdatasets; foreach my $dataset (@datasets) { chomp $dataset; - if ( $path =~ /^$dataset/ ) { push @matchingdatasets, $dataset; } + my ($mountpoint, $mounted) = ($dataset =~ m/([^\t]*)\t*(.*)/); + if ($mounted ne "yes") { + next; + } + + if ( $path =~ /^$mountpoint/ ) { push @matchingdatasets, $mountpoint; } } my $bestmatch = ''; From 44bcd21f269e17765acd1ad0d45161902a205c7b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:25:38 +0100 Subject: [PATCH 32/42] don't use hardcoded paths --- findoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/findoid b/findoid index 48301a4..0b5a00b 100755 --- a/findoid +++ b/findoid @@ -8,7 +8,7 @@ use strict; use warnings; -my $zfs = '/sbin/zfs'; +my $zfs = 'zfs'; my %args = getargs(@ARGV); my $progversion = '1.4.7'; From b748b27a0d4a3da71095dd18bcd75a2988e78e61 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:30:43 +0100 Subject: [PATCH 33/42] handle FileNotFound errors properly --- findoid | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/findoid b/findoid index 48301a4..41558fb 100755 --- a/findoid +++ b/findoid @@ -64,6 +64,10 @@ sub getversions { my $filename = "$dataset/$snappath/$snap/$relpath"; my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($filename); + if (!defined $size) { + next; + } + # only push to the $versions hash if this size and mtime aren't already present (simple dedupe) my $duplicate = 0; foreach my $version (keys %versions) { From 838222e500c9e01f44501219cd8a8d1c85d6161f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:33:43 +0100 Subject: [PATCH 34/42] also show current file version if available --- findoid | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/findoid b/findoid index 48301a4..108481c 100755 --- a/findoid +++ b/findoid @@ -77,6 +77,14 @@ sub getversions { } } + my $filename = "$dataset/$relpath"; + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($filename); + + if (defined $size) { + $versions{$filename}{'size'} = $size; + $versions{$filename}{'mtime'} = $mtime; + } + return %versions; } From db0e83019f92850f19a7a9c7ea0e9fbb09dfc262 Mon Sep 17 00:00:00 2001 From: Dan Langille Date: Thu, 21 Nov 2019 11:27:15 -0500 Subject: [PATCH 35/42] remove 's in monitoring messages fixes #461 - untested --- sanoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 2b566ef..22357c7 100755 --- a/sanoid +++ b/sanoid @@ -159,16 +159,16 @@ sub monitor_snapshots { if ($elapsed == -1) { push @msgs, "CRIT: $path has no $type snapshots at all!"; } else { - push @msgs, "CRIT: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispcrit)"; + push @msgs, "CRIT: $path newest $type snapshot is $dispelapsed old (should be < $dispcrit)"; } } } elsif ($elapsed > $warn) { if ($warn > 0) { if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; } - push @msgs, "WARN: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; + push @msgs, "WARN: $path newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; } } else { - # push @msgs .= "OK: $path\'s newest $type snapshot is $dispelapsed old \n"; + # push @msgs .= "OK: $path newest $type snapshot is $dispelapsed old \n"; } } From 22a7445404cc6917db994eec46912733d1f3cea6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 28 Dec 2019 01:59:05 +0100 Subject: [PATCH 36/42] Revert "Depend on systemd" The debian package ships with a systemd timer unit but can still be usefull on systems without systemd This reverts commit 59e181e61d2e0d9f29834fc65b2201f75d1490a4. --- packages/debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index da70b65..d154147 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -12,7 +12,6 @@ Package: sanoid Architecture: all Depends: libcapture-tiny-perl, libconfig-inifiles-perl, - systemd, zfsutils-linux | zfs, ${misc:Depends}, ${perl:Depends} From d7edf8ddff92e9e2693e2b6e92de188093dc555b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 28 Dec 2019 02:15:28 +0100 Subject: [PATCH 37/42] remove invalid locks caused by race conditions or else the block the critical function of snapshot taking until manual intervention --- sanoid | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 2b566ef..def7feb 100755 --- a/sanoid +++ b/sanoid @@ -1389,7 +1389,9 @@ sub checklock { # make sure lockfile contains something if ( -z $lockfile) { # zero size lockfile, something is wrong - die "ERROR: something is wrong! $lockfile is empty\n"; + warn "WARN: deleting invalid/empty $lockfile\n"; + unlink $lockfile; + return 1 } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if @@ -1400,7 +1402,9 @@ sub checklock { close FH; # if we didn't get exactly 2 items from the lock file there is a problem if (scalar(@lock) != 2) { - die "ERROR: $lockfile is invalid.\n" + warn "WARN: deleting invalid $lockfile\n" + unlink $lockfile; + return 1 } my $lockmutex = pop(@lock); From 1dc77c20acc77b4ce085c9c4b759a1330637653c Mon Sep 17 00:00:00 2001 From: Havard Date: Tue, 31 Dec 2019 14:36:49 +0100 Subject: [PATCH 38/42] Update INSTALL.md Removed invalid reference to sanoid.conf.example. Tried to make the last sentence clearer. --- INSTALL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 8f0af5d..bc56558 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -175,4 +175,6 @@ pkg install p5-Config-Inifiles p5-Capture-Tiny pv mbuffer lzop ## Sanoid -Take a look at the files `sanoid.defaults.conf` and` sanoid.conf.example` for all possible configuration options. Also have a look at the README.md +Take a look at the files `sanoid.defaults.conf` and `sanoid.conf` for all possible configuration options. + +Also have a look at the README.md for a simpler suggestion for `sanoid.conf`. From 30fb5aabeb6f545c1879c843cd538ea22caeb94b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 13 Jan 2020 19:25:22 +0100 Subject: [PATCH 39/42] implemented fallback for listing snapshots on solaris --- syncoid | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f891099..c013745 100755 --- a/syncoid +++ b/syncoid @@ -1459,7 +1459,10 @@ sub getsnaps() { if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; - close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; + close FH or do { + # fallback (solaris for example doesn't support the -t option) + return getsnapsfallback($type,$rhost,$fs,$isroot,%snaps); + }; # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. @@ -1510,6 +1513,87 @@ sub getsnaps() { return %snaps; } +sub getsnapsfallback() { + # fallback (solaris for example doesn't support the -t option) + my ($type,$rhost,$fs,$isroot,%snaps) = @_; + my $mysudocmd; + my $fsescaped = escapeshellparam($fs); + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; + if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } + open FH, $getsnapcmd; + my @rawsnaps = ; + close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; + + my %creationtimes=(); + + my $state = 0; + foreach my $line (@rawsnaps) { + if ($state < 0) { + $state++; + next; + } + + if ($state eq 0) { + if ($line !~ /\Q$fs\E\@.*type\s*snapshot/) { + # skip non snapshot type object + $state = -2; + next; + } + } elsif ($state eq 1) { + if ($line !~ /\Q$fs\E\@.*guid/) { + die "CRITICAL ERROR: snapshots couldn't be listed for $fs (guid parser error)"; + } + + chomp $line; + my $guid = $line; + $guid =~ s/^.*\tguid\t*(\d*).*/$1/; + my $snap = $line; + $snap =~ s/^.*\@(.*)\tguid.*$/$1/; + $snaps{$type}{$snap}{'guid'}=$guid; + } elsif ($state eq 2) { + if ($line !~ /\Q$fs\E\@.*creation/) { + die "CRITICAL ERROR: snapshots couldn't be listed for $fs (creation parser error)"; + } + + chomp $line; + my $creation = $line; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; + my $snap = $line; + $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; + + # the accuracy of the creation timestamp is only for a second, but + # snapshots in the same second are highly likely. The list command + # has an ordered output so we append another three digit running number + # to the creation timestamp and make sure those are ordered correctly + # for snapshot with the same creation timestamp + my $counter = 0; + my $creationsuffix; + while ($counter < 999) { + $creationsuffix = sprintf("%s%03d", $creation, $counter); + if (!defined $creationtimes{$creationsuffix}) { + $creationtimes{$creationsuffix} = 1; + last; + } + $counter += 1; + } + + $snaps{$type}{$snap}{'creation'}=$creationsuffix; + } + + $state++; + } + + return %snaps; +} + sub getbookmarks() { my ($rhost,$fs,$isroot,%bookmarks) = @_; my $mysudocmd; From 77fd555a84c2511496427d123bc3e12dd76e3d98 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 01:26:44 +0100 Subject: [PATCH 40/42] fixed state reset --- syncoid | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncoid b/syncoid index c013745..1e7fc4c 100755 --- a/syncoid +++ b/syncoid @@ -1527,6 +1527,7 @@ sub getsnapsfallback() { } my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; + warn "snapshot listing failed, trying fallback command" if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -1586,6 +1587,7 @@ sub getsnapsfallback() { } $snaps{$type}{$snap}{'creation'}=$creationsuffix; + $state = -1; } $state++; From 3b18948f29c268275de6047e8f638561887abff9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 17:49:43 +0100 Subject: [PATCH 41/42] typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 1e7fc4c..9ae3990 100755 --- a/syncoid +++ b/syncoid @@ -1527,7 +1527,7 @@ sub getsnapsfallback() { } my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; - warn "snapshot listing failed, trying fallback command" + warn "snapshot listing failed, trying fallback command"; if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; From 28ef311ba5b337a9e0283088825cd92f0dfbe01b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 18:10:20 +0100 Subject: [PATCH 42/42] only print stderr output of failed listing command with --debug flag --- syncoid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 9ae3990..ddab82c 100755 --- a/syncoid +++ b/syncoid @@ -1455,8 +1455,13 @@ sub getsnaps() { $fsescaped = escapeshellparam($fsescaped); } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped |"; - if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get A-Hpd 1 -t snapshot guid,creation $fsescaped"; + if ($debug) { + $getsnapcmd = "$getsnapcmd |"; + print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; + } else { + $getsnapcmd = "$getsnapcmd 2>/dev/null |"; + } open FH, $getsnapcmd; my @rawsnaps = ; close FH or do {