From 44f7cae115e5e16d593d771bb6997dffb66f1212 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Thu, 13 Feb 2020 01:01:09 +0100 Subject: [PATCH 01/36] implemented pre pruning script hook --- sanoid | 21 +++++++++++++++++++-- sanoid.conf | 2 ++ sanoid.defaults.conf | 1 + 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index c280d46..f9dfe33 100755 --- a/sanoid +++ b/sanoid @@ -316,6 +316,25 @@ sub prune_snapshots { if (checklock('sanoid_pruning')) { writelock('sanoid_pruning'); foreach my $snap( @prunesnaps ){ + my $dataset = (split '@', $snap)[0]; + my $snapname = (split '@', $snap)[1]; + + if (! $args{'readonly'} && $config{$dataset}{'pre_pruning_script'}) { + $ENV{'SANOID_TARGET'} = $dataset; + $ENV{'SANOID_SNAPNAME'} = $snapname; + if ($args{'verbose'}) { print "executing pre_pruning_script '".$config{$dataset}{'pre_pruning_script'}."' on dataset '$dataset'\n"; } + my $ret = runscript('pre_pruning_script', $dataset); + + delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_SNAPNAME'}; + + if ($ret != 0) { + # warning was already thrown by runscript function + # skip pruning if pre snapshot script returns non zero exit code + next; + } + } + if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; } if (!$args{'force-prune'} && iszfsbusy($path)) { if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; } @@ -323,8 +342,6 @@ sub prune_snapshots { if (! $args{'readonly'}) { if (system($zfs, "destroy", $snap) == 0) { $pruned{$snap} = 1; - my $dataset = (split '@', $snap)[0]; - my $snapname = (split '@', $snap)[1]; if ($config{$dataset}{'pruning_script'}) { $ENV{'SANOID_TARGET'} = $dataset; $ENV{'SANOID_SNAPNAME'} = $snapname; diff --git a/sanoid.conf b/sanoid.conf index 6bd5c62..6d982bc 100644 --- a/sanoid.conf +++ b/sanoid.conf @@ -97,6 +97,8 @@ pre_snapshot_script = /path/to/script.sh ### run script after snapshot post_snapshot_script = /path/to/script.sh + ### run script before pruning snapshot + pre_pruning_script = /path/to/script.sh ### run script after pruning snapshot pruning_script = /path/to/script.sh ### don't take an inconsistent snapshot (skip if pre script fails) diff --git a/sanoid.defaults.conf b/sanoid.defaults.conf index a9ca382..d65601c 100644 --- a/sanoid.defaults.conf +++ b/sanoid.defaults.conf @@ -21,6 +21,7 @@ skip_children = pre_snapshot_script = post_snapshot_script = +pre_pruning_script = pruning_script = script_timeout = 5 no_inconsistent_snapshot = From 6bc210deb3bcffd831b73136d94e3b8d5a75fd24 Mon Sep 17 00:00:00 2001 From: James Bunton Date: Sun, 3 May 2020 23:33:51 +1000 Subject: [PATCH 02/36] fix stream sync from bookmark syncoid fails in the following case: - performing a stream sync (default) - there are no matching snapshots between source and target - there is a bookmark on the source matching a snapshot on the target - there is an unrelated snapshot on the source older than this bookmark This happens because the units for snapshots and bookmarks were different. syncoid tells `zfs send` to generate an incremental stream starting from the bookmark and going to the unrelated older snapshot. ZFS refuses to do this because it doesn't make sense. This fix is a bit of hack. Ideally we should share the counter code between getsnaps and getbookmarks. --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 2eef326..1d7150a 100755 --- a/syncoid +++ b/syncoid @@ -1664,7 +1664,7 @@ sub getbookmarks() { $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; my $bookmark = $line; $bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/; - $bookmarks{$lastguid}{'creation'}=$creation; + $bookmarks{$lastguid}{'creation'}=$creation . "000"; } } From d7bf12623d0d16713c5b5a7d42065e752e3c4753 Mon Sep 17 00:00:00 2001 From: Leo Winter Date: Tue, 6 Oct 2020 13:48:34 +0000 Subject: [PATCH 03/36] Add git to apt install on Debian --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 57f225b..dcc7e34 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -21,7 +21,7 @@ Install prerequisite software: ```bash -apt install debhelper libcapture-tiny-perl libconfig-inifiles-perl pv lzop mbuffer +apt install debhelper libcapture-tiny-perl libconfig-inifiles-perl pv lzop mbuffer git ``` From 309c0866fa8dc7793846e607015cc627dcd98350 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 7 Dec 2020 22:40:28 +0100 Subject: [PATCH 04/36] implemented removal of conflicting snapshots with force-delete option --- README.md | 4 +- syncoid | 32 ++++++++++++- tests/syncoid/8_force_delete_snapshot/run.sh | 48 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100755 tests/syncoid/8_force_delete_snapshot/run.sh diff --git a/README.md b/README.md index b4558c0..5a052e9 100644 --- a/README.md +++ b/README.md @@ -336,11 +336,11 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --force-delete - Remove target datasets recursively (WARNING: this will also affect child datasets with matching snapshots/bookmarks), if there are no matching snapshots/bookmarks. + Remove target datasets recursively (WARNING: this will also affect child datasets with matching snapshots/bookmarks), if there are no matching snapshots/bookmarks. Also removes conflicting snapshots if the replication would fail because of a snapshot which has the same name between source and target but different contents. + --no-clone-handling - This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + This argument tells syncoid to not recreate clones on the target on initial sync and doing a normal replication instead. + --dumpsnaps diff --git a/syncoid b/syncoid index b771d8f..7d8329b 100755 --- a/syncoid +++ b/syncoid @@ -813,6 +813,36 @@ sub syncdataset { if ($exitcode < 2) { $exitcode = 2; } return 0; } + } elsif ($args{'force-delete'} && $stdout =~ /\Qdestination already exists\E/) { + (my $existing) = $stdout =~ m/^cannot restore to ([^:]*): destination already exists$/g; + if ($existing eq "") { + warn "CRITICAL ERROR: $synccmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + } + + if (!$quiet) { print "WARN: removing existing destination: $existing\n"; } + my $rcommand = ''; + my $mysudocmd = ''; + my $existingescaped = escapeshellparam($existing); + + if ($targethost ne '') { $rcommand = "$sshcmd $targethost"; } + if (!$targetisroot) { $mysudocmd = $sudocmd; } + + my $prunecmd = "$mysudocmd $zfscmd destroy $existingescaped; "; + if ($targethost ne '') { + $prunecmd = escapeshellparam($prunecmd); + } + + my $ret = system("$rcommand $prunecmd"); + if ($ret != 0) { + warn "CRITICAL ERROR: $rcommand $prunecmd failed: $?"; + if ($exitcode < 2) { $exitcode = 2; } + return 0; + } else { + # redo sync and skip snapshot creation (already taken) + return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs, undef, 1); + } } else { warn "CRITICAL ERROR: $synccmd failed: $?"; if ($exitcode < 2) { $exitcode = 2; } @@ -1994,4 +2024,4 @@ Options: --no-clone-handling Don't try to recreate clones on target --no-privilege-elevation Bypass the root check, for use with ZFS permission delegation - --force-delete Remove target datasets recursively, if there are no matching snapshots/bookmarks + --force-delete Remove target datasets recursively, if there are no matching snapshots/bookmarks (also overwrites conflicting named snapshots) diff --git a/tests/syncoid/8_force_delete_snapshot/run.sh b/tests/syncoid/8_force_delete_snapshot/run.sh new file mode 100755 index 0000000..899092a --- /dev/null +++ b/tests/syncoid/8_force_delete_snapshot/run.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# test replication with deletion of conflicting snapshot on target + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-8.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-8" +TARGET_CHECKSUM="ee439200c9fa54fc33ce301ef64d4240a6c5587766bfeb651c5cf358e11ec89d -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@duplicate + +# initial replication +../../../syncoid -r --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# recreate snapshot with the same name on src +zfs destroy "${POOL_NAME}"/src@duplicate +zfs snapshot "${POOL_NAME}"/src@duplicate +sleep 1 +../../../syncoid -r --force-delete --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output1=$(zfs list -t snapshot -r -H -o guid,name "${POOL_NAME}"/src | sed 's/@syncoid_.*$'/@syncoid_/) +checksum1=$(echo "${output1}" | shasum -a 256) + +output2=$(zfs list -t snapshot -r -H -o guid,name "${POOL_NAME}"/dst | sed 's/@syncoid_.*$'/@syncoid_/ | sed 's/dst/src/') +checksum2=$(echo "${output2}" | shasum -a 256) + +if [ "${checksum1}" != "${checksum2}" ]; then + exit 1 +fi + +exit 0 From 316b01e9c2e0cc92a826178dbd39136f0d54dded Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 9 Dec 2020 09:46:54 +0100 Subject: [PATCH 05/36] fix duplicate key definitions by only using the first occurence and printing a warning --- sanoid | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sanoid b/sanoid index 041445c..56d11bc 100755 --- a/sanoid +++ b/sanoid @@ -881,6 +881,13 @@ sub init { if (! defined ($defaults{'template_default'}{$key})) { die "FATAL ERROR: I don't understand the setting $key you've set in \[$section\] in $conf_file.\n"; } + + # in case of duplicate lines we will end up with an array of all values + my $value = $ini{$section}{$key}; + if (ref($value) eq 'ARRAY') { + warn "duplicate key '$key' in section '$section', using the value from the first occurence and ignoring the others.\n"; + $ini{$section}{$key} = $value->[0]; + } } if ($section =~ /^template_/) { next; } # don't process templates directly From 48eefd21e85dfdf8f41193295ee3175099c199c2 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 14 Dec 2020 20:57:34 +0100 Subject: [PATCH 06/36] don't try to parse stats for spares which are in use as there are none --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 041445c..1972bda 100755 --- a/sanoid +++ b/sanoid @@ -1239,7 +1239,7 @@ sub check_zpool() { ## no display for verbose level 1 next if ($verbose==1); ## don't display working devices for verbose level 2 - if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL" || $sta eq "INUSE")) { + if ($verbose==2 && ($state eq "OK" || $sta eq "ONLINE" || $sta eq "AVAIL")) { # check for io/checksum errors my @vdeverr = (); From c151703837ba618944fb3779dccaa43d2d869d42 Mon Sep 17 00:00:00 2001 From: "git@end.re" Date: Thu, 8 Apr 2021 15:01:45 +0200 Subject: [PATCH 07/36] Implementing support for ssh_config(5) files --- syncoid | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/syncoid b/syncoid index 3f112de..c0f98e8 100755 --- a/syncoid +++ b/syncoid @@ -20,9 +20,9 @@ my $pvoptions = "-p -t -e -r -b"; # Blank defaults to use ssh client's default # TODO: Merge into a single "sshflags" option? -my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); +my %args = ('sshconfig' => '', 'sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => ''); GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "sendoptions=s", "recvoptions=s", - "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", + "source-bwlimit=s", "target-bwlimit=s", "sshconfig=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-clone-rollback", "no-rollback", "create-bookmark", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", @@ -108,6 +108,9 @@ if (length $args{'sshcipher'}) { if (length $args{'sshport'}) { $args{'sshport'} = "-p $args{'sshport'}"; } +if (length $args{'sshconfig'}) { + $args{'sshconfig'} = "-F $args{'sshconfig'}"; +} if (length $args{'sshkey'}) { $args{'sshkey'} = "-i $args{'sshkey'}"; } @@ -125,7 +128,7 @@ if (length $args{'identifier'}) { } # figure out if source and/or target are remote. -$sshcmd = "$sshcmd $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; +$sshcmd = "$sshcmd $args{'sshconfig'} $args{'sshcipher'} $sshoptions $args{'sshport'} $args{'sshkey'}"; if ($debug) { print "DEBUG: SSHCMD: $sshcmd\n"; } my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); @@ -1978,6 +1981,7 @@ Options: --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ... --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ... + --sshconfig=FILE Specifies an ssh_config(5) file to be used --sshkey=FILE Specifies a ssh key to use to connect --sshport=PORT Connects to remote on a particular port --sshcipher|c=CIPHER Passes CIPHER to ssh to use a particular cipher set From 3954008697333592226a7c6b817b22d522e60b04 Mon Sep 17 00:00:00 2001 From: Gordon Schulz Date: Sat, 10 Apr 2021 12:59:31 +0200 Subject: [PATCH 08/36] Fix bogus dates in %changelog to satisfy rpmlint - 11/24/2020 -> Tuesday - 12/04/2018 -> Tuesday Prior to changes: ``` $ rpmlint sanoid.spec sanoid.spec: E: specfile-error warning: bogus date in %changelog: Wed Nov 24 2020 Christoph Klaffl - 2.1.0 sanoid.spec: E: specfile-error warning: bogus date in %changelog: Wed Dec 04 2018 Christoph Klaffl - 2.0.0 0 packages and 1 specfiles checked; 2 errors, 0 warnings. ``` After changes: ``` $ rpmlint sanoid.spec 0 packages and 1 specfiles checked; 0 errors, 0 warnings. ``` --- packages/rhel/sanoid.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index b4452e8..376f58a 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -111,13 +111,13 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog -* Wed Nov 24 2020 Christoph Klaffl - 2.1.0 +* Tue Nov 24 2020 Christoph Klaffl - 2.1.0 - Bump to 2.1.0 * Wed Oct 02 2019 Christoph Klaffl - 2.0.3 - Bump to 2.0.3 * Wed Sep 25 2019 Christoph Klaffl - 2.0.2 - Bump to 2.0.2 -* Wed Dec 04 2018 Christoph Klaffl - 2.0.0 +* Tue Dec 04 2018 Christoph Klaffl - 2.0.0 - Bump to 2.0.0 * Sat Apr 28 2018 Dominic Robinson - 1.4.18-1 - Bump to 1.4.18 From c2f3f5b6b16bdba642ce6f2a7b3d73ceab1b59a9 Mon Sep 17 00:00:00 2001 From: Andrew Berry Date: Thu, 15 Apr 2021 07:05:03 -0400 Subject: [PATCH 09/36] Fix missing t on target --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b4558c0..9a2425a 100644 --- a/README.md +++ b/README.md @@ -340,7 +340,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --no-clone-handling - This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + This argument tells syncoid to not recreate clones on the target on initial sync and doing a normal replication instead. + --dumpsnaps From e0862dff5234e8cad075782dc293de39ff19a15c Mon Sep 17 00:00:00 2001 From: Alex Haydock Date: Sun, 2 May 2021 15:08:28 +0100 Subject: [PATCH 10/36] Improve documentation for RHEL-family distros --- INSTALL.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..4891b3c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -46,20 +46,28 @@ sudo systemctl enable sanoid.timer sudo systemctl start sanoid.timer ``` -## CentOS +## CentOS/RHEL Install prerequisite software: ```bash -# Install and enable epel if we don't already have it, and git too +# Install and enable EPEL if we don't already have it, and git too: +# (Note that on RHEL we cannot enable EPEL with the epel-release +# package, so you should follow the instructions on the main EPEL site.) sudo yum install -y epel-release git +# On CentOS, we also need to enable the PowerTools repo: +sudo yum config-manager --set-enabled powertools +# On RHEL, instead of PowerTools, we need to enable the CodeReady Builder repo: +sudo subscription-manager repos --enable=codeready-builder-for-rhel-8-x86_64-rpms # Install the packages that Sanoid depends on: -sudo yum install -y perl-Config-IniFiles perl-Data-Dumper perl-Capture-Tiny lzop mbuffer mhash pv -# if the perl dependencies can't be found in the configured repositories you can install them from CPAN manually: +sudo yum install -y perl-Config-IniFiles perl-Data-Dumper perl-Capture-Tiny perl-Getopt-Long lzop mbuffer mhash pv +# The repositories above should contain all the relevant Perl modules, but if you +# still cannot find them then you can install them from CPAN manually: sudo dnf install perl-CPAN perl-CPAN -cpan # answer the questions and past the following lines +cpan # answer the questions and paste the following lines: # install Capture::Tiny # install Config::IniFiles +# install Getopt::Long ``` Clone this repo, then put the executables and config files into the appropriate directories: From c125835e4433c6d5e852b1e06b0fa710dcb695f9 Mon Sep 17 00:00:00 2001 From: Gerhard Roethlin Date: Fri, 23 Jul 2021 17:39:28 +0200 Subject: [PATCH 11/36] MacOS Install Information Installation information for running sanoid on MacOS via launchd. A launchd example plist file is provided in the same way equivalent files are created on other platforms. --- INSTALL.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..5e4a6e3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -171,6 +171,51 @@ For Alpine Linux this can be done with: `apk --no-cache add procps` +## MacOS + +Install prerequisite software: + +``` +perl -MCPAN -e install Config::IniFiles +``` + +The crontab can be used as on a normal unix. To use launchd instead, this example config file can be use can be used. Modify it for your needs. In particular, adjust the sanoid path. +It will start sanoid once per hour, at minute 51. Missed invocations due to standby will be merged into a single invocation at the next wakeup. + +```bash +cat << "EOF" | sudo tee /Library/LaunchDaemons/net.openoid.Sanoid.plist + + + + + Label + net.openoid.Sanoid + ProgramArguments + + /usr/local/sanoid/sanoid + --cron + + EnvironmentVariables + + TZ + UTC + PATH + /usr/local/zfs/bin:$PATH:/usr/local/bin + + StartCalendarInterval + + + Minute + 51 + + + + +EOF + +sudo launchctl load /Library/LaunchDaemons/net.openoid.Sanoid.plist +``` + ## Other OSes **Sanoid** depends on the Perl module Config::IniFiles and will not operate without it. Config::IniFiles may be installed from CPAN, though the project strongly recommends using your distribution's repositories instead. From a9ece1c7bfc934e689cf87cf4dfb56e4e13a9287 Mon Sep 17 00:00:00 2001 From: Daryl Bennett Date: Mon, 23 Aug 2021 20:22:35 -0400 Subject: [PATCH 12/36] Include syncoid remote user requirment Add requirement for syncoid w/ remote hosts to be able to sudo without a password. Recommends a least privilege configuration option for enabling `NOPASSWD` --- INSTALL.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..68b2fa9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -206,3 +206,12 @@ Adapt the timer interval to the lowest configured snapshot interval. Take a look at the files `sanoid.defaults.conf` and `sanoid.conf` for all possible configuration options. Also have a look at the README.md for a simpler suggestion for `sanoid.conf`. + +## Syncoid +If you are pushing or pulling from a remote host, create an user with privldeges to `ssh` as well as `sudo`. To ensure that `zfs send/recieve` can execute, adjust the privileges of the user to execute `sudo` **without** a password for only the `zfs` binary. Modify `/etc/sudoers` by running `# visudo`. Add the following line for your user. + +``` +... + ALL=NOPASSWD: /usr/local/sbin/zfs +... +``` From 523d5006092157094efaea1669ea24d208cdcfde Mon Sep 17 00:00:00 2001 From: Lilis Iskandar Date: Wed, 29 Sep 2021 18:19:05 +0800 Subject: [PATCH 13/36] Update README.md Signed-off-by: Lilis Iskandar --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21432ac..3859ed9 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

-More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job but see INSTALL.md fore more details: +More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job but see INSTALL.md for more details: ``` * * * * * TZ=UTC /usr/local/bin/sanoid --cron ``` -`Note`: Using UTC as timezone is recommend to prevent problems with daylight saving times +`Note`: Using UTC as timezone is recommended to prevent problems with daylight saving times And its /etc/sanoid/sanoid.conf might look something like this: From 864ab7f95b67cf92a34731e08de61fd59cf3a5bc Mon Sep 17 00:00:00 2001 From: Ryan David Sheasby Date: Tue, 2 Nov 2021 11:36:13 +0200 Subject: [PATCH 14/36] Update debian installation instructions Previous instructions couldn't actually be copied and pasted. This will update the instructions to work correctly when copied as-is --- INSTALL.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..338cddb 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -22,28 +22,26 @@ Install prerequisite software: ```bash -apt install debhelper libcapture-tiny-perl libconfig-inifiles-perl pv lzop mbuffer build-essential +apt install debhelper libcapture-tiny-perl libconfig-inifiles-perl pv lzop mbuffer build-essential git ``` Clone this repo, build the debian package and install it (alternatively you can skip the package and do it manually like described below for CentOS): ```bash -# Download the repo as root to avoid changing permissions later -sudo git clone https://github.com/jimsalterjrs/sanoid.git +git clone https://github.com/jimsalterjrs/sanoid.git cd sanoid # checkout latest stable release or stay on master for bleeding edge stuff (but expect bugs!) git checkout $(git tag | grep "^v" | tail -n 1) ln -s packages/debian . dpkg-buildpackage -uc -us -apt install ../sanoid_*_all.deb +sudo apt install ../sanoid_*_all.deb ``` Enable sanoid timer: ```bash # enable and start the sanoid timer -sudo systemctl enable sanoid.timer -sudo systemctl start sanoid.timer +sudo systemctl enable --now sanoid.timer ``` ## CentOS From 1fcf34800138833b1efbcb64276047f7b6fe042c Mon Sep 17 00:00:00 2001 From: Topslakr <67913692+Topslakr@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:57:18 -0500 Subject: [PATCH 15/36] Update Install.md for Centos 8 Updated the install language, and commands, to better server Centos 8 and it's derivatives. Instead of using CPAN, enabling the 'PowerTools' repo will make those modules available for installation and kept current with normal OS updates. Updated command to enable _and_ start sanoid.timer, instead of using two commands. --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..e4724d8 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -53,9 +53,11 @@ Install prerequisite software: ```bash # Install and enable epel if we don't already have it, and git too sudo yum install -y epel-release git +# For Centos 8 you need to enable the PowerTools repo to make all the needed Perl modules available (Recommended) +sudo dnf config-manager --set-enabled powertools # Install the packages that Sanoid depends on: sudo yum install -y perl-Config-IniFiles perl-Data-Dumper perl-Capture-Tiny lzop mbuffer mhash pv -# if the perl dependencies can't be found in the configured repositories you can install them from CPAN manually: +# if you don't want to enable the powertools repo, the perl dependencies can be installed from CPAN manually: sudo dnf install perl-CPAN perl-CPAN cpan # answer the questions and past the following lines # install Capture::Tiny @@ -142,8 +144,7 @@ sudo systemctl daemon-reload # Enable sanoid-prune.service to allow it to be triggered by sanoid.service sudo systemctl enable sanoid-prune.service # Enable and start the Sanoid timer -sudo systemctl enable sanoid.timer -sudo systemctl start sanoid.timer +sudo systemctl enable --now sanoid.timer ``` Now, proceed to configure [**Sanoid**](#configuration) From c408bddab4e0eb11b16628d9cd52240dd323cec7 Mon Sep 17 00:00:00 2001 From: Daryl Bennett Date: Mon, 23 Aug 2021 20:22:35 -0400 Subject: [PATCH 16/36] Include syncoid remote user requirment Add requirement for syncoid w/ remote hosts to be able to sudo without a password. Recommends a least privilege configuration option for enabling `NOPASSWD` --- INSTALL.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index c8411d3..ec7645b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -206,3 +206,12 @@ Adapt the timer interval to the lowest configured snapshot interval. Take a look at the files `sanoid.defaults.conf` and `sanoid.conf` for all possible configuration options. Also have a look at the README.md for a simpler suggestion for `sanoid.conf`. + +## Syncoid +If you are pushing or pulling from a remote host, create a user with privileges to `ssh` as well as `sudo`. To ensure that `zfs send/receive` can execute, adjust the privileges of the user to execute `sudo` **without** a password for only the `zfs` binary (run `which zfs` to find the path of the `zfs` binary). Modify `/etc/sudoers` by running `# visudo`. Add the following line for your user. + +``` +... + ALL=NOPASSWD: +... +``` From fa67fbe845977ea136c724c4c8680bf1102b30b7 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Mon, 12 Sep 2022 23:44:52 +0200 Subject: [PATCH 17/36] bring back no-rollback option --- syncoid | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index ec6ae9d..1746009 100755 --- a/syncoid +++ b/syncoid @@ -24,7 +24,7 @@ my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r", "sendoptions=s", "recvoptions=s", "source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@", "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", - "no-clone-handling", "no-privilege-elevation", "force-delete", "create-bookmark", + "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size) or pod2usage(2); @@ -288,8 +288,11 @@ sub syncdataset { my $sourcefsescaped = escapeshellparam($sourcefs); my $targetfsescaped = escapeshellparam($targetfs); - # keep forcedrecv as a variable to allow us to disable it with an optional argument later if necessary + # if no rollbacks are allowed, disable forced receive my $forcedrecv = "-F"; + if (defined $args{'no-rollback'}) { + $forcedrecv = ""; + } if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; } @@ -1956,6 +1959,7 @@ Options: --keep-sync-snap Don't destroy created sync snapshots --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) --preserve-recordsize Preserves the recordsize on initial sends to the target + --no-rollback Does not rollback snapshots on target (it probably requires a readonly target) --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times --sendoptions=OPTIONS Use advanced options for zfs send (the arguments are filtered as needed), e.g. syncoid --sendoptions="Lc e" sets zfs send -L -c -e ... --recvoptions=OPTIONS Use advanced options for zfs receive (the arguments are filtered as needed), e.g. syncoid --recvoptions="ux recordsize o compression=lz4" sets zfs receive -u -x recordsize -o compression=lz4 ... From 4f150ec6ec90844c9b8d6aa4eaee6202734bc993 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Sep 2022 00:02:03 +0200 Subject: [PATCH 18/36] fix snapshot listing fallback --- syncoid | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index ec6ae9d..23271d5 100755 --- a/syncoid +++ b/syncoid @@ -1503,6 +1503,8 @@ sub getsnaps() { my $fsescaped = escapeshellparam($fs); if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + my $rhostOriginal = $rhost; + if ($rhost ne '') { $rhost = "$sshcmd $rhost"; # double escaping needed @@ -1520,7 +1522,7 @@ sub getsnaps() { my @rawsnaps = ; close FH or do { # fallback (solaris for example doesn't support the -t option) - return getsnapsfallback($type,$rhost,$fs,$isroot,%snaps); + return getsnapsfallback($type,$rhostOriginal,$fs,$isroot,%snaps); }; # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines From a5d7896c0fe5ad6feb8c2969bbecb8d28155fab0 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Sep 2022 00:16:49 +0200 Subject: [PATCH 19/36] improve accuracy of zfs receive check --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index ec6ae9d..1a945f9 100755 --- a/syncoid +++ b/syncoid @@ -1113,7 +1113,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($debug) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs *(receive|recv).*\Q$fs\E\Z/) { + if ($process =~ /zfs *(receive|recv)[^\/]*\Q$fs\E\Z/) { # there's already a zfs receive process for our target filesystem - return true if ($debug) { print "DEBUG: process $process matches target $fs!\n"; } return 1; From 0808575874875ee202d4a035744db75ca9d54162 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 13 Sep 2022 17:13:01 +0200 Subject: [PATCH 20/36] syncoid should exit with an error if the specified src dataset doesn't exist (and only skip child datasets which vanished) --- syncoid | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/syncoid b/syncoid index ec6ae9d..167fc58 100755 --- a/syncoid +++ b/syncoid @@ -141,6 +141,8 @@ my %avail = checkcommands(); my %snaps; my $exitcode = 0; +my $replicationCount = 0; + ## break here to call replication individually so that we ## ## can loop across children separately, for recursive ## ## replication ## @@ -297,7 +299,7 @@ sub syncdataset { if (!defined $sync) { # zfs already printed the corresponding error - if ($error =~ /\bdataset does not exist\b/) { + if ($error =~ /\bdataset does not exist\b/ && $replicationCount > 0) { if (!$quiet) { print "WARN Skipping dataset (dataset no longer exists): $sourcefs...\n"; } return 0; } @@ -811,6 +813,8 @@ sub syncdataset { } } + $replicationCount++; + if (defined $args{'no-sync-snap'}) { if (defined $args{'create-bookmark'}) { my $bookmarkcmd; From 5d469c4bb2ec5e6e18073d32e99d5d1bde77a948 Mon Sep 17 00:00:00 2001 From: joelishness Date: Mon, 3 Oct 2022 21:36:26 -0700 Subject: [PATCH 21/36] Update README.md Fixed some typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21432ac..eaa8b38 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --no-clone-handling - This argument tells syncoid to not recreate clones on the targe on initial sync and doing a normal replication instead. + This argument tells syncoid to not recreate clones on the target on initial sync, and do a normal replication instead. + --dumpsnaps From 9f898439e0150d8abe7f8390ee86ac2b8cfca889 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:47 -0500 Subject: [PATCH 22/36] spelling: a lot Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21432ac..629a1c8 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ For more full details on sanoid.conf settings see [Wiki page](https://github.com + --debug - This prints out quite alot of additional information during a sanoid run, and is normally not needed. + This prints out quite a lot of additional information during a sanoid run, and is normally not needed. + --readonly @@ -374,7 +374,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --debug - This prints out quite alot of additional information during a sanoid run, and is normally not needed. + This prints out quite a lot of additional information during a sanoid run, and is normally not needed. + --help From 7d24d97045a91cc477c758b4498767eb6b021ee0 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:07 -0500 Subject: [PATCH 23/36] spelling: available Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CHANGELIST | 2 +- packages/debian/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 49fcd50..78fcd0f 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -2,7 +2,7 @@ [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) [syncoid] use sudo if neccessary for checking pool capabilities regarding resumeable send (@phreaker0) - [syncoid] catch another case were the resume state isn't availabe anymore (@phreaker0) + [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) [syncoid] append timezone offset to the syncoid snapshot name to fix DST collisions (@phreaker0) diff --git a/packages/debian/changelog b/packages/debian/changelog index b394acb..b4e3f6c 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -4,7 +4,7 @@ sanoid (2.1.0) unstable; urgency=medium [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) [syncoid] use sudo if neccessary for checking pool capabilities regarding resumeable send (@phreaker0) - [syncoid] catch another case were the resume state isn't availabe anymore (@phreaker0) + [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) [syncoid] append timezone offset to the syncoid snapshot name to fix DST collisions (@phreaker0) From cad12155018216d42fc06b33b00275dd8f73162a Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:07 -0500 Subject: [PATCH 24/36] spelling: debugging Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 13ea085..06712a4 100755 --- a/sanoid +++ b/sanoid @@ -1137,7 +1137,7 @@ sub check_zpool() { } } - # Tony: Debuging + # Tony: Debugging # print "Size: $size \t Used: $used \t Avai: $avail \t Cap: $cap \t Health: $health\n"; close(STAT); From c37f412797424993a2cce5b54b86cfb5da8d99ea Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:35:48 -0500 Subject: [PATCH 25/36] spelling: errlevel Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sanoid | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sanoid b/sanoid index 06712a4..81a7eac 100755 --- a/sanoid +++ b/sanoid @@ -130,7 +130,7 @@ sub monitor_snapshots { my ($config, $snaps, $snapsbytype, $snapsbypath) = @_; my %datestamp = get_date(); - my $errorlevel = 0; + my $errlevel = 0; my $msg; my @msgs; my @paths; @@ -169,7 +169,7 @@ sub monitor_snapshots { my $dispcrit = displaytime($crit); if ( $elapsed > $crit || $elapsed == -1) { if ($crit > 0) { - if (! $config{$section}{'monitor_dont_crit'}) { $errorlevel = 2; } + if (! $config{$section}{'monitor_dont_crit'}) { $errlevel = 2; } if ($elapsed == -1) { push @msgs, "CRIT: $path has no $type snapshots at all!"; } else { @@ -178,7 +178,7 @@ sub monitor_snapshots { } } elsif ($elapsed > $warn) { if ($warn > 0) { - if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; } + if (! $config{$section}{'monitor_dont_warn'} && ($errlevel < 2) ) { $errlevel = 1; } push @msgs, "WARN: $path newest $type snapshot is $dispelapsed old (should be < $dispwarn)"; } } else { @@ -196,7 +196,7 @@ sub monitor_snapshots { if ($msg eq '') { $msg = "OK: all monitored datasets \($paths\) have fresh snapshots"; } print "$msg\n"; - exit $errorlevel; + exit $errlevel; } From 12e962a133996f1b30d200b11a507bb820c10a3c Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:36:04 -0500 Subject: [PATCH 26/36] spelling: errors Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CHANGELIST | 2 +- packages/debian/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 78fcd0f..259ee3c 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -29,7 +29,7 @@ 2.0.2 [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) [sanoid] changed and simplified DST handling (@shodanshok) [syncoid] reset partially resume state automatically (@phreaker0) - [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) + [syncoid] handle some zfs errors automatically by parsing the stderr outputs (@phreaker0) [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) [syncoid] don't use hardcoded paths (@phreaker0) [syncoid] fix for special setup with listsnapshots=on (@phreaker0) diff --git a/packages/debian/changelog b/packages/debian/changelog index b4e3f6c..096f2ad 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -39,7 +39,7 @@ sanoid (2.0.2) unstable; urgency=medium [overall] documentation updates, new dependencies, small fixes, more warnings (@benyanke, @matveevandrey, @RulerOf, @klemens-u, @johnramsden, @danielewood, @g-a-c, @hartzell, @fryfrog, @phreaker0) [syncoid] changed and simplified DST handling (@shodanshok) [syncoid] reset partially resume state automatically (@phreaker0) - [syncoid] handle some zfs erros automatically by parsing the stderr outputs (@phreaker0) + [syncoid] handle some zfs errors automatically by parsing the stderr outputs (@phreaker0) [syncoid] fixed ordering of snapshots with the same creation timestamp (@phreaker0) [syncoid] don't use hardcoded paths (@phreaker0) [syncoid] fix for special setup with listsnapshots=on (@phreaker0) From c43048598a09d9fd6522f5a0bdaf2a7da925fe1c Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:18:24 -0500 Subject: [PATCH 27/36] spelling: github Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CHANGELIST | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 259ee3c..fbee5c1 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -118,12 +118,12 @@ replicating to target/parent/child2. This could still use some cleanup TBH; syncoid SHOULD exit 3 if any of these errors happen (to assist detection of errors in scripting) but now would exit 0. -1.4.12 Sanoid now strips trailing whitespace in template definitions in sanoid.conf, per Github #61 +1.4.12 Sanoid now strips trailing whitespace in template definitions in sanoid.conf, per GitHub #61 1.4.11 enhanced Syncoid to use zfs `guid` property rather than `creation` property to ensure snapshots on source and target actually match. This immediately prevents conflicts due to timezone differences on source and target, and also paves the way in the future for Syncoid to find matching snapshots even after `zfs rename` on source - or target. Thank you Github user @mailinglists35 for the idea! + or target. Thank you GitHub user @mailinglists35 for the idea! 1.4.10 added --compress=pigz-fast and --compress=pigz-slow. On a Xeon E3-1231v3, pigz-fast is equivalent compression to --compress=gzip but with compressed throughput of 75.2 MiB/s instead of 18.1 MiB/s. pigz-slow is around 5% @@ -241,4 +241,4 @@ 1.0.1 ported slightly modified iszfsbusy sub from syncoid to sanoid (to keep from thinning snapshots during replications) -1.0.0 initial commit to Github +1.0.0 initial commit to GitHub From a6664247423351b6d96a899b9376648769b0e6b5 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:38:09 -0500 Subject: [PATCH 28/36] spelling: mbytes Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 629a1c8..1674ed9 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --target-bwlimit - This is the bandwidth limit in bytes (kbytes, mbytesm etc) per second imposed upon the target. This is mainly used if the source does not have mbuffer installed, but bandwidth limits are desired. + This is the bandwidth limit in bytes (kbytes, mbytes, etc) per second imposed upon the target. This is mainly used if the source does not have mbuffer installed, but bandwidth limits are desired. + --no-command-checks From 9421892402b45aacefc6284b12c2aa88bfd12ac7 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:08 -0500 Subject: [PATCH 29/36] spelling: naming Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index ec6ae9d..e7a9194 100755 --- a/syncoid +++ b/syncoid @@ -821,7 +821,7 @@ sub syncdataset { } if ($debug) { print "DEBUG: $bookmarkcmd\n"; } system($bookmarkcmd) == 0 or do { - # fallback: assume nameing conflict and try again with guid based suffix + # fallback: assume naming conflict and try again with guid based suffix my $guid = $snaps{'source'}{$newsyncsnap}{'guid'}; $guid = substr($guid, 0, 6); From 93f2b8863e037bb945d6b5cf30d278276eeba2d6 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:08 -0500 Subject: [PATCH 30/36] spelling: necessary Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CHANGELIST | 2 +- packages/debian/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index fbee5c1..8c3d76e 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,7 +1,7 @@ 2.1.0 [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) - [syncoid] use sudo if neccessary for checking pool capabilities regarding resumeable send (@phreaker0) + [syncoid] use sudo if necessary for checking pool capabilities regarding resumeable send (@phreaker0) [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) diff --git a/packages/debian/changelog b/packages/debian/changelog index 096f2ad..3796a37 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -3,7 +3,7 @@ sanoid (2.1.0) unstable; urgency=medium [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) - [syncoid] use sudo if neccessary for checking pool capabilities regarding resumeable send (@phreaker0) + [syncoid] use sudo if necessary for checking pool capabilities regarding resumeable send (@phreaker0) [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) From 7a8b0adb6d91c6e9d114dc41c138959bf1b08aad Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:08 -0500 Subject: [PATCH 31/36] spelling: overridden Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sanoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid b/sanoid index 81a7eac..4dd1937 100755 --- a/sanoid +++ b/sanoid @@ -889,7 +889,7 @@ sub init { # for sections directly when they've already been defined recursively, without starting them over from scratch. if (! defined ($config{$section}{'initialized'})) { if ($args{'debug'}) { print "DEBUG: initializing \$config\{$section\} with default values from $default_conf_file.\n"; } - # set default values from %defaults, which can then be overriden by template + # set default values from %defaults, which can then be overridden by template # and/or local settings within the module. foreach my $key (keys %{$defaults{'template_default'}}) { if (! ($key =~ /template|recursive|children_only/)) { From 3ffa57c72fec37e1b3bd8ed43489bd5b19174779 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:09 -0500 Subject: [PATCH 32/36] spelling: resumable Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CHANGELIST | 4 ++-- README.md | 4 ++-- packages/debian/changelog | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 8c3d76e..6ddc565 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,7 +1,7 @@ 2.1.0 [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) - [syncoid] use sudo if necessary for checking pool capabilities regarding resumeable send (@phreaker0) + [syncoid] use sudo if necessary for checking pool capabilities regarding resumable send (@phreaker0) [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) @@ -84,7 +84,7 @@ [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) [syncoid] Added support for ZStandard compression.(@danielewood) [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumable send with datasets containing spaces (@phreaker0) [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) diff --git a/README.md b/README.md index 1674ed9..4e890ff 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ syncoid root@remotehost:data/images/vm backup/images/vm Which would pull-replicate the filesystem from the remote host to the local system over an SSH tunnel. Syncoid supports recursive replication (replication of a dataset and all its child datasets) and uses mbuffer buffering, lzop compression, and pv progress bars if the utilities are available on the systems used. -If ZFS supports resumeable send/receive streams on both the source and target those will be enabled as default. +If ZFS supports resumable send/receive streams on both the source and target those will be enabled as default. As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature. @@ -334,7 +334,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --no-resume - This argument tells syncoid to not use resumeable zfs send/receive streams. + This argument tells syncoid to not use resumable zfs send/receive streams. + --force-delete diff --git a/packages/debian/changelog b/packages/debian/changelog index 3796a37..4cab69b 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -3,7 +3,7 @@ sanoid (2.1.0) unstable; urgency=medium [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) - [syncoid] use sudo if necessary for checking pool capabilities regarding resumeable send (@phreaker0) + [syncoid] use sudo if necessary for checking pool capabilities regarding resumable send (@phreaker0) [syncoid] catch another case were the resume state isn't available anymore (@phreaker0) [syncoid] check for an invalid argument combination (@phreaker0) [syncoid] fix iszfsbusy check for similar dataset names (@phreaker0) @@ -102,7 +102,7 @@ sanoid (2.0.0) unstable; urgency=medium [sanoid] implemented monitor-capacity flag for checking zpool capacity limits (@phreaker0) [syncoid] Added support for ZStandard compression.(@danielewood) [syncoid] implemented support for excluding datasets from replication with regular expressions (@phreaker0) - [syncoid] correctly parse zfs column output, fixes resumeable send with datasets containing spaces (@phreaker0) + [syncoid] correctly parse zfs column output, fixes resumable send with datasets containing spaces (@phreaker0) [syncoid] added option for using extra identification in the snapshot name for replication to multiple targets (@phreaker0) [syncoid] added option for skipping the parent dataset in recursive replication (@phreaker0) [syncoid] typos (@UnlawfulMonad, @jsavikko, @phreaker0) From 0fcaab54f603e70bc1da14fa0c62376784e5d285 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:09 -0500 Subject: [PATCH 33/36] spelling: snapshotted Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e890ff..5f7794c 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Will be executed before the snapshot(s) of a single dataset are taken. The follo | ----------------- | ----------- | | `SANOID_SCRIPT` | The type of script being executed, one of `pre`, `post`, or `prune`. Allows for one script to be used for multiple tasks | | `SANOID_TARGET` | **DEPRECATED** The dataset about to be snapshot (only the first dataset will be provided) | -| `SANOID_TARGETS` | Comma separated list of all datasets to be snapshoted (currently only a single dataset, multiple datasets will be possible later with atomic groups) | +| `SANOID_TARGETS` | Comma separated list of all datasets to be snapshotted (currently only a single dataset, multiple datasets will be possible later with atomic groups) | | `SANOID_SNAPNAME` | **DEPRECATED** The name of the snapshot that will be taken (only the first name will be provided, does not include the dataset name) | | `SANOID_SNAPNAMES` | Comma separated list of all snapshot names that will be taken (does not include the dataset name) | | `SANOID_TYPES` | Comma separated list of all snapshot types to be taken (yearly, monthly, weekly, daily, hourly, frequently) | From 2333f1142d4bab0985f8a997a1d61a8f5d20c26c Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:30:10 -0500 Subject: [PATCH 34/36] spelling: suppress Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f7794c..1f3df88 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ For more full details on sanoid.conf settings see [Wiki page](https://github.com + --quiet - Supress non-error output. + Suppress non-error output. + --verbose @@ -370,7 +370,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --quiet - Supress non-error output. + Suppress non-error output. + --debug From ae28c10982212d80b3f7565df6e91723d03c89c3 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 1 Jan 2023 14:42:58 -0500 Subject: [PATCH 35/36] spelling: want Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/common/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 904c98f..9c88eff 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -34,7 +34,7 @@ function checkEnvironment { echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "you should be running this test in a" echo "dedicated vm, as it will mess with your system!" - echo "Are you sure you wan't to continue? (y)" + echo "Are you sure you want to continue? (y)" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" set -x From 38f2d625a881d1cfb37e830fe4f769d8c2f608d5 Mon Sep 17 00:00:00 2001 From: dodexahedron Date: Thu, 2 Feb 2023 19:32:08 -0700 Subject: [PATCH 36/36] Clarified that compression is on the wire --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21432ac..4b0c197 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --compress - Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, xz, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + Compression method to use for network transfer. Currently accepted options: gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, xz, lzo (default) & none. If the selected compression method is unavailable on the source and destination, no compression will be used. + --source-bwlimit