From fd71e794ad4e56afa54784516fd2c2e709d1723f Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Thu, 16 May 2019 11:38:39 -0500 Subject: [PATCH 01/25] Gracefully handle error when source dataset disappeared If the source dataset dissappeared before we were able to get the sanoid:sync property, do not set an error exit code. Handle this gracefully as this should not be considered an error. Fixes #380 --- syncoid | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 21e9d1e..98403ec 100755 --- a/syncoid +++ b/syncoid @@ -290,7 +290,10 @@ sub syncdataset { if (!defined $sync) { # zfs already printed the corresponding error - if ($exitcode < 2) { $exitcode = 2; } + if (dataset_exists($sourcehost, $sourcefs, $sourceisroot) and $exitcode < 2) { + $exitcode = 2; + } + return 0; } @@ -1147,6 +1150,37 @@ sub getzfsvalue { return $value; } +sub dataset_exists { + my ($rhost, $fs, $isroot) = @_; + + my $fsescaped = escapeshellparam($fs); + + my $mysudocmd; + if ($isroot) { + $mysudocmd = ''; + } + else { + $mysudocmd = $sudocmd; + } + + my $command = "$rhost $mysudocmd $zfscmd get -H creation $fsescaped"; + + if ($debug) { + print "$command\n"; + } + + open my $fh, '-|', "$command 2>&1"; + my $result = <$fh>; + close $fh; + + if ($result =~ /dataset does not exist/) { + warn "$fsescaped does not exist...\n"; + return 0; + } + + return 1; +} + sub readablebytes { my $bytes = shift; my $disp; From 784efe2d85f05e4bb3ca8f060ec1cfefc902b48c Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Thu, 16 May 2019 11:57:08 -0500 Subject: [PATCH 02/25] Remove debug warning --- syncoid | 1 - 1 file changed, 1 deletion(-) diff --git a/syncoid b/syncoid index 98403ec..c443b84 100755 --- a/syncoid +++ b/syncoid @@ -1174,7 +1174,6 @@ sub dataset_exists { close $fh; if ($result =~ /dataset does not exist/) { - warn "$fsescaped does not exist...\n"; return 0; } From 3892d73594c1e9faf14d65f5d0008095a005d98d Mon Sep 17 00:00:00 2001 From: Michael Schout Date: Wed, 29 May 2019 17:13:31 -0500 Subject: [PATCH 03/25] Capture output in getzfsvalue and handle datasets that disappeared Change getzfsvalue() so that if called in array context, returns the value and the error as a two element list. This allows the caller to recieve the error from the zfs command. If the dataset went away before fetching the sanoid:sync property, just issue a warning and skip it. --- syncoid | 55 ++++++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/syncoid b/syncoid index c443b84..f7a4b66 100755 --- a/syncoid +++ b/syncoid @@ -286,12 +286,18 @@ sub syncdataset { if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } - my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); + my ($sync, $error) = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync'); if (!defined $sync) { # zfs already printed the corresponding error - if (dataset_exists($sourcehost, $sourcefs, $sourceisroot) and $exitcode < 2) { - $exitcode = 2; + if ($error =~ /\bdataset does not exist\b/) { + if (!$quiet) { print "WARN Skipping dataset (dataset no longer exists): $sourcefs...\n"; } + return 0; + } + else { + # print the error out and set exit code + print "ERROR: $error\n"; + if ($exitcode < 2) { $exitcode = 2 } } return 0; @@ -1137,47 +1143,22 @@ sub getzfsvalue { my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } if ($debug) { print "$rhost $mysudocmd $zfscmd get -H $property $fsescaped\n"; } - open FH, "$rhost $mysudocmd $zfscmd get -H $property $fsescaped |"; - my $value = ; - close FH; - - if (!defined $value) { - return undef; - } + my ($value, $error, $exit) = capture { + system("$rhost $mysudocmd $zfscmd get -H $property $fsescaped"); + }; my @values = split(/\t/,$value); $value = $values[2]; - return $value; -} -sub dataset_exists { - my ($rhost, $fs, $isroot) = @_; + my $wantarray = wantarray || 0; - my $fsescaped = escapeshellparam($fs); - - my $mysudocmd; - if ($isroot) { - $mysudocmd = ''; - } - else { - $mysudocmd = $sudocmd; + # If we are in scalar context and there is an error, print it out. + # Otherwise we assume the caller will deal with it. + if (!$wantarray and $error) { + print "ERROR getzfsvalue $fs $property: $error\n"; } - my $command = "$rhost $mysudocmd $zfscmd get -H creation $fsescaped"; - - if ($debug) { - print "$command\n"; - } - - open my $fh, '-|', "$command 2>&1"; - my $result = <$fh>; - close $fh; - - if ($result =~ /dataset does not exist/) { - return 0; - } - - return 1; + return $wantarray ? ($value, $error) : $value; } sub readablebytes { From dee9d1817fa113f5cb68fbec6d64d32818475720 Mon Sep 17 00:00:00 2001 From: "Gabriel A. Devenyi" Date: Fri, 14 Jun 2019 11:50:34 -0400 Subject: [PATCH 04/25] Add ability to configure pv --- syncoid | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/syncoid b/syncoid index 29b7319..326f614 100755 --- a/syncoid +++ b/syncoid @@ -16,6 +16,7 @@ use Sys::Hostname; use Capture::Tiny ':all'; my $mbuffer_size = "16M"; +my $pvoptions = "-p -t -e -r -b"; # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? @@ -24,7 +25,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback", - "create-bookmark", + "create-bookmark", "pv-options=s" => \$pvoptions, "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -1225,13 +1226,13 @@ sub buildsynccmd { } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $bwlimit $mbufferoptions |"; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd $pvoptions -s $pvsize |"; } $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. #$synccmd = "$sendcmd | $pvcmd | $compressargs{'cmd'} | $mbuffercmd | $sshcmd $targethost '$compressargs{'decomcmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; - if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd $pvoptions -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $compressargs{'cmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost "; @@ -1254,7 +1255,7 @@ sub buildsynccmd { $synccmd .= " | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. @@ -1268,7 +1269,7 @@ sub buildsynccmd { $synccmd .= " | "; if ($avail{'compress'}) { $synccmd .= "$compressargs{'decomcmd'} | "; } - if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd $pvoptions -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$compressargs{'cmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost "; @@ -1791,6 +1792,7 @@ Options: --source-bwlimit= Bandwidth limit in bytes/kbytes/etc per second on the source transfer --target-bwlimit= Bandwidth limit in bytes/kbytes/etc per second on the target transfer --mbuffer-size=VALUE Specify the mbuffer size (default: 16M), please refer to mbuffer(1) manual page. + --pv-options=OPTIONS Configure how pv displays the progress bar, default '-p -t -e -r -b' --no-stream Replicates using newest snapshot instead of intermediates --no-sync-snap Does not create new snapshot, only transfers existing --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) From a85ad93a18c6a8873a7dd8b2c0bd36d66c5d9031 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 18 Jun 2019 08:32:11 +0200 Subject: [PATCH 05/25] added needed verbose flag for send size estimation (at least for latest FreeBSD codebase) --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 975d7c5..ad62930 100755 --- a/syncoid +++ b/syncoid @@ -1602,9 +1602,9 @@ sub getsendsize { if (defined($receivetoken)) { $sendoptions = getoptionsline(\@sendoptions, ('e')); } else { - $sendoptions = getoptionsline(\@sendoptions, ('D','L','R','c','e','h','p','v','w')); + $sendoptions = getoptionsline(\@sendoptions, ('D','L','R','c','e','h','p','w')); } - my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nP $snaps"; + my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send $sendoptions -nvP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } open FH, "$getsendsizecmd 2>&1 |"; From 96a48efd92a2949761349f480dc049624ad5bcd5 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 2 Oct 2019 16:42:38 +0200 Subject: [PATCH 06/25] prepare v2.0.3 bugfix release --- CHANGELIST | 4 +++- VERSION | 2 +- packages/debian/changelog | 6 ++++++ packages/rhel/sanoid.spec | 4 +++- sanoid | 2 +- syncoid | 2 +- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index e66940d..b83b717 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,5 +1,7 @@ +2.0.3 [sanoid] reverted DST handling and improved it as quickfix (@phreaker0) + 2.0.2 [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) - [syncoid] changed and simplified DST handling (@shodanshok) + [sanoid] changed and simplified DST handling (@shodanshok) [syncoid] reset partially resume state automatically (@phreaker0) [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) diff --git a/VERSION b/VERSION index e9307ca..50ffc5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.2 +2.0.3 diff --git a/packages/debian/changelog b/packages/debian/changelog index 829e530..7ea878b 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,9 @@ +sanoid (2.0.3) unstable; urgency=medium + + [sanoid] reverted DST handling and improved it as quickfix (@phreaker0) + + -- Jim Salter Wed, 02 Oct 2019 17:00:00 +0100 + sanoid (2.0.2) unstable; urgency=medium [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 3aff0a9..ce912ea 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 2.0.2 +%global version 2.0.3 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Wed Oct 02 2019 Christoph Klaffl - 2.0.3 +- Bump to 2.0.3 * Wed Sep 25 2019 Christoph Klaffl - 2.0.2 - Bump to 2.0.2 * Wed Dec 04 2018 Christoph Klaffl - 2.0.0 diff --git a/sanoid b/sanoid index a17b91d..2b566ef 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.2'; +$::VERSION = '2.0.3'; my $MINIMUM_DEFAULTS_VERSION = 2; use strict; diff --git a/syncoid b/syncoid index e048391..f891099 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.0.2'; +$::VERSION = '2.0.3'; use strict; use warnings; From a1de743c3f02c009a6de2873735f07c1e26ce162 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 16 Oct 2019 09:08:17 +0200 Subject: [PATCH 07/25] fix sha512 checksum for gentoo packaging --- packages/gentoo/sys-fs/sanoid/Manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gentoo/sys-fs/sanoid/Manifest b/packages/gentoo/sys-fs/sanoid/Manifest index d25575a..06ef31c 100644 --- a/packages/gentoo/sys-fs/sanoid/Manifest +++ b/packages/gentoo/sys-fs/sanoid/Manifest @@ -1,4 +1,4 @@ AUX sanoid.cron 45 BLAKE2B 3f6294bbbf485dc21a565cd2c8da05a42fb21cdaabdf872a21500f1a7338786c60d4a1fd188bbf81ce85f06a376db16998740996f47c049707a5109bdf02c052 SHA512 7676b32f21e517e8c84a097c7934b54097cf2122852098ea756093ece242125da3f6ca756a6fbb82fc348f84b94bfd61639e86e0bfa4bbe7abf94a8a4c551419 -DIST sanoid-2.0.2.tar.gz 115797 BLAKE2B d00a038062df3dd8e77d3758c7b80ed6da0bac4931fb6df6adb72eeddb839c63d5129e0a281948a483d02165dad5a8505e1a55dc851360d3b366371038908142 SHA512 9d999b0f071bc3c3ca956df11e1501fd72a842f7d3315ede3ab3b5e0a36351100b6edbab8448bba65a2e187e4e8f77ff24671ed33b28f2fca9bb6ad0801aba9d +DIST sanoid-2.0.2.tar.gz 115797 BLAKE2B d00a038062df3dd8e77d3758c7b80ed6da0bac4931fb6df6adb72eeddb839c63d5129e0a281948a483d02165dad5a8505e1a55dc851360d3b366371038908142 SHA512 73e3d25dbdd58a78ffc4384584304e7230c5f31a660ce6d2a9b9d52a92a3796f1bc25ae865dbc74ce586cbd6169dbb038340f4a28e097e77ab3eb192b15773db EBUILD sanoid-2.0.2.ebuild 796 BLAKE2B f3d633289d66c60fd26cb7731bc6b63533019f527aaec9ca8e5c0e748542d391153dbb55b17b8c981ca4fa4ae1fc8dc202b5480c13736fca250940b3b5ebb793 SHA512 d0143680c029ffe4ac37d97a979ed51527b4b8dd263d0c57e43a4650bf8a9bb8 EBUILD sanoid-9999.ebuild 776 BLAKE2B 416b8d04a9e5a84bce46d2a6f88eaefe03804944c03bc7f49b7a5b284b844212a6204402db3de3afa5d9c0545125d2631e7231c8cb2a3537bdcb10ea1be46b6a SHA512 98d8a30a13e75d7847ae9d60797d54078465bf75c6c6d9b6fd86075e342c0374 From 0e5c2e1cff7cfe5dcaf893f9c8fa838c3cf22b8a Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:19:56 +0100 Subject: [PATCH 08/25] improve dataset detection by only including mounted datasets --- findoid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/findoid b/findoid index 48301a4..67b5de6 100755 --- a/findoid +++ b/findoid @@ -102,14 +102,19 @@ sub getdataset { my ($path) = @_; - open FH, "$zfs list -Ho mountpoint |"; + open FH, "$zfs list -H -t filesystem -o mountpoint,mounted |"; my @datasets = ; close FH; my @matchingdatasets; foreach my $dataset (@datasets) { chomp $dataset; - if ( $path =~ /^$dataset/ ) { push @matchingdatasets, $dataset; } + my ($mountpoint, $mounted) = ($dataset =~ m/([^\t]*)\t*(.*)/); + if ($mounted ne "yes") { + next; + } + + if ( $path =~ /^$mountpoint/ ) { push @matchingdatasets, $mountpoint; } } my $bestmatch = ''; From 44bcd21f269e17765acd1ad0d45161902a205c7b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:25:38 +0100 Subject: [PATCH 09/25] don't use hardcoded paths --- findoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/findoid b/findoid index 48301a4..0b5a00b 100755 --- a/findoid +++ b/findoid @@ -8,7 +8,7 @@ use strict; use warnings; -my $zfs = '/sbin/zfs'; +my $zfs = 'zfs'; my %args = getargs(@ARGV); my $progversion = '1.4.7'; From b748b27a0d4a3da71095dd18bcd75a2988e78e61 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:30:43 +0100 Subject: [PATCH 10/25] handle FileNotFound errors properly --- findoid | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/findoid b/findoid index 48301a4..41558fb 100755 --- a/findoid +++ b/findoid @@ -64,6 +64,10 @@ sub getversions { my $filename = "$dataset/$snappath/$snap/$relpath"; my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($filename); + if (!defined $size) { + next; + } + # only push to the $versions hash if this size and mtime aren't already present (simple dedupe) my $duplicate = 0; foreach my $version (keys %versions) { From 838222e500c9e01f44501219cd8a8d1c85d6161f Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 5 Nov 2019 17:33:43 +0100 Subject: [PATCH 11/25] also show current file version if available --- findoid | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/findoid b/findoid index 48301a4..108481c 100755 --- a/findoid +++ b/findoid @@ -77,6 +77,14 @@ sub getversions { } } + my $filename = "$dataset/$relpath"; + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($filename); + + if (defined $size) { + $versions{$filename}{'size'} = $size; + $versions{$filename}{'mtime'} = $mtime; + } + return %versions; } From db0e83019f92850f19a7a9c7ea0e9fbb09dfc262 Mon Sep 17 00:00:00 2001 From: Dan Langille Date: Thu, 21 Nov 2019 11:27:15 -0500 Subject: [PATCH 12/25] remove 's in monitoring messages fixes #461 - untested --- sanoid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sanoid b/sanoid index 2b566ef..22357c7 100755 --- a/sanoid +++ b/sanoid @@ -159,16 +159,16 @@ sub monitor_snapshots { if ($elapsed == -1) { push @msgs, "CRIT: $path has no $type snapshots at all!"; } else { - push @msgs, "CRIT: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispcrit)"; + push @msgs, "CRIT: $path newest $type snapshot is $dispelapsed old (should be < $dispcrit)"; } } } elsif ($elapsed > $warn) { if ($warn > 0) { if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; } - push @msgs, "WARN: $path\'s newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; + push @msgs, "WARN: $path newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; } } else { - # push @msgs .= "OK: $path\'s newest $type snapshot is $dispelapsed old \n"; + # push @msgs .= "OK: $path newest $type snapshot is $dispelapsed old \n"; } } From a1f5e4c0c006e16a5047a16fc65c9b3663adb81e Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Wed, 4 Dec 2019 22:19:29 -0500 Subject: [PATCH 13/25] Add cache-dir option. --- README.md | 4 ++++ sanoid | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bc8ed83..d5e143d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da Specify a location for the config file named sanoid.conf. Defaults to /etc/sanoid ++ --cache-dir + + Specify a directory to store the zfs snapshot cache. Defaults to /var/cache/sanoid + + --take-snapshots This will process your sanoid.conf file, create snapshots, but it will NOT purge expired ones. (Note that snapshots taken are atomic in an individual dataset context, not a global context - snapshots of pool/dataset1 and pool/dataset2 will each be internally consistent and atomic, but one may be a few filesystem transactions "newer" than the other.) diff --git a/sanoid b/sanoid index 2b566ef..fd39f3a 100755 --- a/sanoid +++ b/sanoid @@ -11,15 +11,19 @@ use strict; use warnings; use Config::IniFiles; # read samba-style conf file use Data::Dumper; # debugging - print contents of hash -use File::Path; # for rmtree command in use_prune +use File::Path 'make_path'; use Getopt::Long qw(:config auto_version auto_help); use Pod::Usage; # pod2usage use Time::Local; # to parse dates in reverse use Capture::Tiny ':all'; -my %args = ("configdir" => "/etc/sanoid"); +my %args = ( + "configdir" => "/etc/sanoid", + "cache-dir" => "/var/cache/sanoid" +); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "monitor-health", "force-update", "configdir=s", + "configdir=s", "cache-dir=s", + "monitor-health", "force-update", "monitor-snapshots", "take-snapshots", "prune-snapshots", "force-prune", "monitor-capacity" ) or pod2usage(2); @@ -41,9 +45,13 @@ my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; # parse config file my %config = init($conf_file,$default_conf_file); +my $cache_dir = $args{'cache-dir'}; + +make_path($cache_dir); + # if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL my $forcecacheupdate = 0; -my $cache = '/var/cache/sanoidsnapshots.txt'; +my $cache = "$cache_dir/snapshots.txt"; my $cacheTTL = 900; # 15 minutes my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate ); my %pruned; @@ -1659,6 +1667,7 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif Options: --configdir=DIR Specify a directory to find config file sanoid.conf + --cache-dir=DIR Specify a directory to store the zfs snapshot cache --cron Creates snapshots and purges expired snapshots --verbose Prints out additional information during a sanoid run From 59a07f92b4920952cc9137b03c1533656f48b121 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 6 Dec 2019 14:28:37 -0500 Subject: [PATCH 14/25] Add run-dir option. --- README.md | 4 ++++ sanoid | 24 ++++++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d5e143d..c2b502b 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da Specify a directory to store the zfs snapshot cache. Defaults to /var/cache/sanoid ++ --run-dir + + Specify a directory for temporary files such as lock files. Defaults to /var/run/sanoid + + --take-snapshots This will process your sanoid.conf file, create snapshots, but it will NOT purge expired ones. (Note that snapshots taken are atomic in an individual dataset context, not a global context - snapshots of pool/dataset1 and pool/dataset2 will each be internally consistent and atomic, but one may be a few filesystem transactions "newer" than the other.) diff --git a/sanoid b/sanoid index fd39f3a..7f9c2fa 100755 --- a/sanoid +++ b/sanoid @@ -19,10 +19,11 @@ use Capture::Tiny ':all'; my %args = ( "configdir" => "/etc/sanoid", - "cache-dir" => "/var/cache/sanoid" + "cache-dir" => "/var/cache/sanoid", + "run-dir" => "/var/run/sanoid" ); GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet", - "configdir=s", "cache-dir=s", + "configdir=s", "cache-dir=s", "run-dir=s", "monitor-health", "force-update", "monitor-snapshots", "take-snapshots", "prune-snapshots", "force-prune", "monitor-capacity" @@ -46,8 +47,10 @@ my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; my %config = init($conf_file,$default_conf_file); my $cache_dir = $args{'cache-dir'}; +my $run_dir = $args{'run-dir'}; make_path($cache_dir); +make_path($run_dir); # if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL my $forcecacheupdate = 0; @@ -1373,10 +1376,10 @@ sub get_zpool_capacity { sub checklock { # take argument $lockname. # - # read /var/run/$lockname.lock for a pid on first line and a mutex on second line. + # read $run_dir/$lockname.lock for a pid on first line and a mutex on second line. # - # check process list to see if the pid from /var/run/$lockname.lock is still active with - # the original mutex found in /var/run/$lockname.lock. + # check process list to see if the pid from $run_dir/$lockname.lock is still active with + # the original mutex found in $run_dir/$lockname.lock. # # return: # 0 if lock is present and valid for another process @@ -1388,7 +1391,7 @@ sub checklock { # my $lockname = shift; - my $lockfile = "/var/run/$lockname.lock"; + my $lockfile = "$run_dir/$lockname.lock"; if (! -e $lockfile) { # no lockfile @@ -1441,11 +1444,11 @@ sub checklock { sub removelock { # take argument $lockname. # - # make sure /var/run/$lockname.lock actually belongs to me (contains my pid and mutex) + # make sure $run_dir/$lockname.lock actually belongs to me (contains my pid and mutex) # and remove it if it does, die if it doesn't. my $lockname = shift; - my $lockfile = "/var/run/$lockname.lock"; + my $lockfile = "$run_dir/$lockname.lock"; if (checklock($lockname) == 2) { unlink $lockfile; @@ -1460,11 +1463,11 @@ sub removelock { sub writelock { # take argument $lockname. # - # write a lockfile to /var/run/$lockname.lock with first line + # write a lockfile to $run_dir/$lockname.lock with first line # being my pid and second line being my mutex. my $lockname = shift; - my $lockfile = "/var/run/$lockname.lock"; + my $lockfile = "$run_dir/$lockname.lock"; # die honorably rather than overwriting a valid, existing lock if (! checklock($lockname)) { @@ -1668,6 +1671,7 @@ Options: --configdir=DIR Specify a directory to find config file sanoid.conf --cache-dir=DIR Specify a directory to store the zfs snapshot cache + --run-dir=DIR Specify a directory for temporary files such as lock files --cron Creates snapshots and purges expired snapshots --verbose Prints out additional information during a sanoid run From 22a7445404cc6917db994eec46912733d1f3cea6 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 28 Dec 2019 01:59:05 +0100 Subject: [PATCH 15/25] Revert "Depend on systemd" The debian package ships with a systemd timer unit but can still be usefull on systems without systemd This reverts commit 59e181e61d2e0d9f29834fc65b2201f75d1490a4. --- packages/debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/debian/control b/packages/debian/control index da70b65..d154147 100644 --- a/packages/debian/control +++ b/packages/debian/control @@ -12,7 +12,6 @@ Package: sanoid Architecture: all Depends: libcapture-tiny-perl, libconfig-inifiles-perl, - systemd, zfsutils-linux | zfs, ${misc:Depends}, ${perl:Depends} From d7edf8ddff92e9e2693e2b6e92de188093dc555b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Sat, 28 Dec 2019 02:15:28 +0100 Subject: [PATCH 16/25] remove invalid locks caused by race conditions or else the block the critical function of snapshot taking until manual intervention --- sanoid | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 2b566ef..def7feb 100755 --- a/sanoid +++ b/sanoid @@ -1389,7 +1389,9 @@ sub checklock { # make sure lockfile contains something if ( -z $lockfile) { # zero size lockfile, something is wrong - die "ERROR: something is wrong! $lockfile is empty\n"; + warn "WARN: deleting invalid/empty $lockfile\n"; + unlink $lockfile; + return 1 } # lockfile exists. read pid and mutex from it. see if it's our pid. if not, see if @@ -1400,7 +1402,9 @@ sub checklock { close FH; # if we didn't get exactly 2 items from the lock file there is a problem if (scalar(@lock) != 2) { - die "ERROR: $lockfile is invalid.\n" + warn "WARN: deleting invalid $lockfile\n" + unlink $lockfile; + return 1 } my $lockmutex = pop(@lock); From 1dc77c20acc77b4ce085c9c4b759a1330637653c Mon Sep 17 00:00:00 2001 From: Havard Date: Tue, 31 Dec 2019 14:36:49 +0100 Subject: [PATCH 17/25] Update INSTALL.md Removed invalid reference to sanoid.conf.example. Tried to make the last sentence clearer. --- INSTALL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 8f0af5d..bc56558 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -175,4 +175,6 @@ pkg install p5-Config-Inifiles p5-Capture-Tiny pv mbuffer lzop ## Sanoid -Take a look at the files `sanoid.defaults.conf` and` sanoid.conf.example` for all possible configuration options. Also have a look at the README.md +Take a look at the files `sanoid.defaults.conf` and `sanoid.conf` for all possible configuration options. + +Also have a look at the README.md for a simpler suggestion for `sanoid.conf`. From 30fb5aabeb6f545c1879c843cd538ea22caeb94b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 13 Jan 2020 19:25:22 +0100 Subject: [PATCH 18/25] implemented fallback for listing snapshots on solaris --- syncoid | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index f891099..c013745 100755 --- a/syncoid +++ b/syncoid @@ -1459,7 +1459,10 @@ sub getsnaps() { if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; - close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; + close FH or do { + # fallback (solaris for example doesn't support the -t option) + return getsnapsfallback($type,$rhost,$fs,$isroot,%snaps); + }; # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines # as though each were an entirely separate get command. @@ -1510,6 +1513,87 @@ sub getsnaps() { return %snaps; } +sub getsnapsfallback() { + # fallback (solaris for example doesn't support the -t option) + my ($type,$rhost,$fs,$isroot,%snaps) = @_; + my $mysudocmd; + my $fsescaped = escapeshellparam($fs); + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; + if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } + open FH, $getsnapcmd; + my @rawsnaps = ; + close FH or die "CRITICAL ERROR: snapshots couldn't be listed for $fs (exit code $?)"; + + my %creationtimes=(); + + my $state = 0; + foreach my $line (@rawsnaps) { + if ($state < 0) { + $state++; + next; + } + + if ($state eq 0) { + if ($line !~ /\Q$fs\E\@.*type\s*snapshot/) { + # skip non snapshot type object + $state = -2; + next; + } + } elsif ($state eq 1) { + if ($line !~ /\Q$fs\E\@.*guid/) { + die "CRITICAL ERROR: snapshots couldn't be listed for $fs (guid parser error)"; + } + + chomp $line; + my $guid = $line; + $guid =~ s/^.*\tguid\t*(\d*).*/$1/; + my $snap = $line; + $snap =~ s/^.*\@(.*)\tguid.*$/$1/; + $snaps{$type}{$snap}{'guid'}=$guid; + } elsif ($state eq 2) { + if ($line !~ /\Q$fs\E\@.*creation/) { + die "CRITICAL ERROR: snapshots couldn't be listed for $fs (creation parser error)"; + } + + chomp $line; + my $creation = $line; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; + my $snap = $line; + $snap =~ s/^.*\@(.*)\tcreation.*$/$1/; + + # the accuracy of the creation timestamp is only for a second, but + # snapshots in the same second are highly likely. The list command + # has an ordered output so we append another three digit running number + # to the creation timestamp and make sure those are ordered correctly + # for snapshot with the same creation timestamp + my $counter = 0; + my $creationsuffix; + while ($counter < 999) { + $creationsuffix = sprintf("%s%03d", $creation, $counter); + if (!defined $creationtimes{$creationsuffix}) { + $creationtimes{$creationsuffix} = 1; + last; + } + $counter += 1; + } + + $snaps{$type}{$snap}{'creation'}=$creationsuffix; + } + + $state++; + } + + return %snaps; +} + sub getbookmarks() { my ($rhost,$fs,$isroot,%bookmarks) = @_; my $mysudocmd; From 77fd555a84c2511496427d123bc3e12dd76e3d98 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 01:26:44 +0100 Subject: [PATCH 19/25] fixed state reset --- syncoid | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncoid b/syncoid index c013745..1e7fc4c 100755 --- a/syncoid +++ b/syncoid @@ -1527,6 +1527,7 @@ sub getsnapsfallback() { } my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; + warn "snapshot listing failed, trying fallback command" if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -1586,6 +1587,7 @@ sub getsnapsfallback() { } $snaps{$type}{$snap}{'creation'}=$creationsuffix; + $state = -1; } $state++; From 3b18948f29c268275de6047e8f638561887abff9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 17:49:43 +0100 Subject: [PATCH 20/25] typo --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 1e7fc4c..9ae3990 100755 --- a/syncoid +++ b/syncoid @@ -1527,7 +1527,7 @@ sub getsnapsfallback() { } my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 type,guid,creation $fsescaped |"; - warn "snapshot listing failed, trying fallback command" + warn "snapshot listing failed, trying fallback command"; if ($debug) { print "DEBUG: FALLBACK, getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; From 28ef311ba5b337a9e0283088825cd92f0dfbe01b Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 16 Jan 2020 18:10:20 +0100 Subject: [PATCH 21/25] only print stderr output of failed listing command with --debug flag --- syncoid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 9ae3990..ddab82c 100755 --- a/syncoid +++ b/syncoid @@ -1455,8 +1455,13 @@ sub getsnaps() { $fsescaped = escapeshellparam($fsescaped); } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped |"; - if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get A-Hpd 1 -t snapshot guid,creation $fsescaped"; + if ($debug) { + $getsnapcmd = "$getsnapcmd |"; + print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; + } else { + $getsnapcmd = "$getsnapcmd 2>/dev/null |"; + } open FH, $getsnapcmd; my @rawsnaps = ; close FH or do { From 2de006072f0dc3b40f4caf2021253b1763079eba Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 17 Jan 2020 01:04:16 +0100 Subject: [PATCH 22/25] fixed missing ; --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 2720f60..ece2380 100755 --- a/sanoid +++ b/sanoid @@ -1402,7 +1402,7 @@ sub checklock { close FH; # if we didn't get exactly 2 items from the lock file there is a problem if (scalar(@lock) != 2) { - warn "WARN: deleting invalid $lockfile\n" + warn "WARN: deleting invalid $lockfile\n"; unlink $lockfile; return 1 } From bb30496d19affedace5180e282f37b39c151ac84 Mon Sep 17 00:00:00 2001 From: croadfeldt Date: Sat, 18 Jan 2020 01:52:12 -0600 Subject: [PATCH 23/25] Fixed Typo in arguments. Removed errant A in zfs list command line arguments. --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index bfbff7d..2eef326 100755 --- a/syncoid +++ b/syncoid @@ -1470,7 +1470,7 @@ sub getsnaps() { $fsescaped = escapeshellparam($fsescaped); } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get A-Hpd 1 -t snapshot guid,creation $fsescaped"; + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot guid,creation $fsescaped"; if ($debug) { $getsnapcmd = "$getsnapcmd |"; print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; From d3309d311f26b029753f98f67212c76c073a0937 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 22 Jan 2020 17:22:44 +0100 Subject: [PATCH 24/25] post install script for debian package to remove old unused snapshot cache file --- packages/debian/postinst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 packages/debian/postinst diff --git a/packages/debian/postinst b/packages/debian/postinst new file mode 100755 index 0000000..d23e6bc --- /dev/null +++ b/packages/debian/postinst @@ -0,0 +1,4 @@ +#!/bin/bash + +# remove old cache file +[ -f /var/cache/sanoidsnapshots.txt ] && rm /var/cache/sanoidsnapshots.txt From dedf35df85b8ab788bc5722cf16ec33ac9ff4220 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 31 Jan 2020 09:13:18 +0100 Subject: [PATCH 25/25] fix debian postinst in the case of non existing legacy cache file --- packages/debian/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/postinst b/packages/debian/postinst index d23e6bc..0d6142f 100755 --- a/packages/debian/postinst +++ b/packages/debian/postinst @@ -1,4 +1,4 @@ #!/bin/bash # remove old cache file -[ -f /var/cache/sanoidsnapshots.txt ] && rm /var/cache/sanoidsnapshots.txt +[ -f /var/cache/sanoidsnapshots.txt ] && rm /var/cache/sanoidsnapshots.txt || true