From 0b582f6200181b09d0fd09d1e797fa848393ba1e Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Wed, 12 Feb 2020 17:48:42 +0100 Subject: [PATCH] group snapshot creation together so pre/post scripts are only run once per dataset and prepare for future atomic grouping feature --- README.md | 107 ++++++++++++++++++++++++++++----------- sanoid | 147 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 162 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index e35ef27..6f631df 100644 --- a/README.md +++ b/README.md @@ -111,48 +111,97 @@ Which would be enough to tell sanoid to take and keep 36 hourly snapshots, 30 da ### Sanoid script hooks -There are 3 scripts which can optionally be executed at various stages in the lifecycle of a snapshot: +There are three script types which can optionally be executed at various stages in the lifecycle of a snapshot: -##### `pre_snapshot_script` +#### `pre_snapshot_script` -This script will be executed before a snapshot is taken. The following environment variables with be passed: +Will be executed before the snapshot(s) of a single dataset are taken. The following environment variables are passed: -| Env vars | Description -| ----------------- | ----------- -| `SANOID_SCRIPT` | The type of script being executed, one of `pre`, `post`, or `prune`. Allows for one script to be used for multiple tasks -| `SANOID_TARGET` | The dataset about to be snapshot -| `SANOID_SNAPNAME` | The name of the snapshot that will be taken (does not include the dataset name) -| `SANOID_TYPE` | The type of snapshot to be taken (yearly, monthly, weekly, daily, hourly, frequently) -| `SANOID_BATCH` | All the snapshots which will be taken against this dataset (does not include the dataset name), joined by commas. Note that not all of the snapshots will have been taken. For example, monthly is taken before weekly, but weekly is still included when `SANOID_TYPE` is monthly. It is guaranteed to take snapshots in ascending frequency: yearly, monthly, ... frequently +| Env vars | Description | +| ----------------- | ----------- | +| `SANOID_SCRIPT` | The type of script being executed, one of `pre`, `post`, or `prune`. Allows for one script to be used for multiple tasks | +| `SANOID_TARGET` | **DEPRECATED** The dataset about to be snapshot (only the first dataset will be provided) | +| `SANOID_TARGETS` | Comma separated list of all datasets to be snapshoted (currently only a single dataset, multiple datasets will be possible later with atomic groups) | +| `SANOID_SNAPNAME` | **DEPRECATED** The name of the snapshot that will be taken (only the first name will be provided, does not include the dataset name) | +| `SANOID_SNAPNAMES` | Comma separated list of all snapshot names that will be taken (does not include the dataset name) | +| `SANOID_TYPES` | Comma separated list of all snapshot types to be taken (yearly, monthly, weekly, daily, hourly, frequently) | -If the script returns a non-zero exit code, the snapshot will not be taken unless `no_inconsistent_snapshot` is false. +If the script returns a non-zero exit code, the snapshot(s) will not be taken unless `no_inconsistent_snapshot` is false. -##### `post_snapshot_script` +#### `post_snapshot_script` -This script will be executed when: +Will be executed when: - The pre-snapshot script succeeded or - The pre-snapshot script failed and `force_post_snapshot_script` is true. -| Env vars | Description -| -------------------- | ----------- -| `SANOID_SCRIPT` | as above | -| `SANOID_TARGET` | as above | -| `SANOID_SNAPNAME` | as above | -| `SANOID_TYPE` | as above | -| `SANOID_BATCH` | as above | +| Env vars | Description | +| -------------------- | ----------- | +| `SANOID_SCRIPT` | as above | +| `SANOID_TARGET` | **DEPRECATED** as above | +| `SANOID_TARGETS` | as above | +| `SANOID_SNAPNAME` | **DEPRECATED** as above | +| `SANOID_SNAPNAMES` | as above | +| `SANOID_TYPES` | as above | | `SANOID_PRE_FAILURE` | This will indicate if the pre-snapshot script failed | +#### `pruning_script` -##### `pruning_script` +Will be executed after a snapshot is successfully deleted. The following environment variables will be passed: -This script will be executed after a snapshot is successfully deleted. The following environment variables will be passed: +| Env vars | Description | +| ----------------- | ----------- | +| `SANOID_SCRIPT` | as above | +| `SANOID_TARGET` | as above | +| `SANOID_SNAPNAME` | as above | -| Env vars | Description -| ----------------- | ----------- -| `SANOID_SCRIPT` | as above | -| `SANOID_TARGET` | as above | -| `SANOID_SNAPNAME` | as above | + +#### example + +**sanoid.conf**: +``` +... +[sanoid-test-0] + use_template = production + recursive = yes + pre_snapshot_script = /tmp/debug.sh + post_snapshot_script = /tmp/debug.sh + pruning_script = /tmp/debug.sh +... +``` + +**verbose sanoid output**: +``` +... +executing pre_snapshot_script '/tmp/debug.sh' on dataset 'sanoid-test-0' +taking snapshot sanoid-test-0@autosnap_2020-02-12_14:49:33_yearly +taking snapshot sanoid-test-0@autosnap_2020-02-12_14:49:33_monthly +taking snapshot sanoid-test-0@autosnap_2020-02-12_14:49:33_daily +taking snapshot sanoid-test-0@autosnap_2020-02-12_14:49:33_hourly +executing post_snapshot_script '/tmp/debug.sh' on dataset 'sanoid-test-0' +... +``` + +**pre script env variables**: +``` +SANOID_SCRIPT=pre +SANOID_TARGET=sanoid-test-0/b/bb +SANOID_TARGETS=sanoid-test-0/b/bb +SANOID_SNAPNAME=autosnap_2020-02-12_14:49:32_yearly +SANOID_SNAPNAMES=autosnap_2020-02-12_14:49:32_yearly,autosnap_2020-02-12_14:49:32_monthly,autosnap_2020-02-12_14:49:32_daily,autosnap_2020-02-12_14:49:32_hourly +SANOID_TYPES=yearly,monthly,daily,hourly +``` + +**post script env variables**: +``` +SANOID_SCRIPT=post +SANOID_TARGET=sanoid-test-0/b/bb +SANOID_TARGETS=sanoid-test-0/b/bb +SANOID_SNAPNAME=autosnap_2020-02-12_14:49:32_yearly +SANOID_SNAPNAMES=autosnap_2020-02-12_14:49:32_yearly,autosnap_2020-02-12_14:49:32_monthly,autosnap_2020-02-12_14:49:32_daily,autosnap_2020-02-12_14:49:32_hourly +SANOID_TYPES=yearly,monthly,daily,hourly +SANOID_PRE_FAILURE=0 +``` ---------- @@ -237,7 +286,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --source-bwlimit - This is the bandwidth limit in bytes (kbytes, mbytes, etc) per second imposed upon the source. This is mainly used if the target does not have mbuffer installed, but bandwidth limits are desired. + This is the bandwidth limit in bytes (kbytes, mbytes, etc) per second imposed upon the source. This is mainly used if the target does not have mbuffer installed, but bandwidth limits are desired. + --target-bw-limit @@ -257,7 +306,7 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup + --create-bookmark - This argument tells syncoid to create a zfs bookmark for the newest snapshot after it got replicated successfully. The bookmark name will be equal to the snapshot name. Only works in combination with the --no-sync-snap option. This can be very useful for irregular replication where the last matching snapshot on the source was already deleted but the bookmark remains so a replication is still possible. + This argument tells syncoid to create a zfs bookmark for the newest snapshot after it got replicated successfully. The bookmark name will be equal to the snapshot name. Only works in combination with the --no-sync-snap option. This can be very useful for irregular replication where the last matching snapshot on the source was already deleted but the bookmark remains so a replication is still possible. + --no-clone-rollback diff --git a/sanoid b/sanoid index 0b41b18..ea2711a 100755 --- a/sanoid +++ b/sanoid @@ -369,7 +369,7 @@ sub take_snapshots { my %datestamp = get_date(); my $forcecacheupdate = 0; - my @newsnaps; + my %newsnapsgroup; # get utc timestamp of the current day for DST check my $daystartUtc = timelocal(0, 0, 0, $datestamp{'mday'}, ($datestamp{'mon'}-1), $datestamp{'year'}); @@ -392,9 +392,8 @@ sub take_snapshots { my $path = $config{$section}{'path'}; my @types = ('yearly','monthly','weekly','daily','hourly','frequently'); - my @batch = (); - foreach my $type (@types) { + foreach my $type (@types) { if ($config{$section}{$type} > 0) { my $newestage; # in seconds @@ -514,44 +513,57 @@ sub take_snapshots { my $maxage = time()-$lastpreferred; if ( $newestage > $maxage ) { - # update to most current possible datestamp - %datestamp = get_date(); # print "we should have had a $type snapshot of $path $maxage seconds ago; most recent is $newestage seconds old.\n"; - my $snap = { - 'dataset' => $path, - 'snapshot' => "autosnap_$datestamp{'sortable'}_$type", - 'recursive' => $config{$section}{'zfs_recursion'}, # use zfs (atomic) recursion if specified in config - 'handleDst' => $handleDst, - 'type' => $type, - 'batch' => \@batch, # Reference the source array, because we may be adding to it in subsequent loops - }; - push(@batch, $snap->{snapshot}); - push(@newsnaps, $snap); + if (!exists $newsnapsgroup{$path}) { + $newsnapsgroup{$path} = { + 'recursive' => $config{$section}{'zfs_recursion'}, + 'handleDst' => $handleDst, + 'datasets' => [$path], # for later atomic grouping, currently only a one element array + 'types' => [] + }; + } + + push(@{$newsnapsgroup{$path}{'types'}}, $type); } } } } - if ( (scalar(@newsnaps)) > 0) { - foreach my $snapData ( @newsnaps ) { - my $dataset = $snapData->{dataset}; - my $snapname = $snapData->{snapshot}; + if (%newsnapsgroup) { + while ((my $path, my $snapData) = each(%newsnapsgroup)) { my $recursiveFlag = $snapData->{recursive}; my $dstHandling = $snapData->{handleDst}; - my $batch = join(",", @{$snapData->{batch}}); + + my @datasets = @{$snapData->{datasets}}; + my $dataset = $datasets[0]; + my @types = @{$snapData->{types}}; + + # same timestamp for all snapshots types (daily, hourly, ...) + my %datestamp = get_date(); + my @snapshots; + + foreach my $type (@types) { + my $snapname = "autosnap_$datestamp{'sortable'}_$type"; + push(@snapshots, $snapname); + } + + my $datasetString = join(",", @datasets); + my $typeString = join(",", @types); + my $snapshotString = join(",", @snapshots); + my $extraMessage = ""; if ($recursiveFlag) { $extraMessage = " (zfs recursive)"; } - my $snap = "$dataset\@$snapname"; my $presnapshotfailure = 0; my $ret = 0; if ($config{$dataset}{'pre_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; - $ENV{'SANOID_SNAPNAME'} = $snapname; - $ENV{'SANOID_TYPE'} = $snapData->{type}; - $ENV{'SANOID_BATCH'} = $batch; + $ENV{'SANOID_TARGETS'} = $datasetString; + $ENV{'SANOID_SNAPNAME'} = @snapshots[0]; + $ENV{'SANOID_SNAPNAMES'} = $snapshotString; + $ENV{'SANOID_TYPES'} = $typeString; $ENV{'SANOID_SCRIPT'} = 'pre'; if ($args{'verbose'}) { print "executing pre_snapshot_script '".$config{$dataset}{'pre_snapshot_script'}."' on dataset '$dataset'\n"; } @@ -560,9 +572,10 @@ sub take_snapshots { } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_TARGETS'}; delete $ENV{'SANOID_SNAPNAME'}; - delete $ENV{'SANOID_TYPE'}; - delete $ENV{'SANOID_BATCH'}; + delete $ENV{'SANOID_SNAPNAMES'}; + delete $ENV{'SANOID_TYPES'}; delete $ENV{'SANOID_SCRIPT'}; if ($ret != 0) { @@ -571,49 +584,56 @@ sub take_snapshots { $presnapshotfailure = 1; } } - if ($args{'verbose'}) { print "taking snapshot $snap$extraMessage\n"; } - if (!$args{'readonly'}) { - my $stderr; - my $exit; - ($stderr, $exit) = tee_stderr { - if ($recursiveFlag) { - system($zfs, "snapshot", "-r", "$snap"); - } else { - system($zfs, "snapshot", "$snap"); - } - }; - $exit == 0 or do { - if ($dstHandling) { - if ($stderr =~ /already exists/) { - $exit = 0; - $snap =~ s/_([a-z]+)$/dst_$1/g; - if ($args{'verbose'}) { print "taking dst snapshot $snap$extraMessage\n"; } - if ($recursiveFlag) { - system($zfs, "snapshot", "-r", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; - } else { - system($zfs, "snapshot", "$snap") == 0 - or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + foreach my $snap (@snapshots) { + $snap = "$dataset\@$snap"; + if ($args{'verbose'}) { print "taking snapshot $snap$extraMessage\n"; } + + if (!$args{'readonly'}) { + my $stderr; + my $exit; + ($stderr, $exit) = tee_stderr { + if ($recursiveFlag) { + system($zfs, "snapshot", "-r", "$snap"); + } else { + system($zfs, "snapshot", "$snap"); + } + }; + + $exit == 0 or do { + if ($dstHandling) { + if ($stderr =~ /already exists/) { + $exit = 0; + $snap =~ s/_([a-z]+)$/dst_$1/g; + if ($args{'verbose'}) { print "taking dst snapshot $snap$extraMessage\n"; } + if ($recursiveFlag) { + system($zfs, "snapshot", "-r", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; + } else { + system($zfs, "snapshot", "$snap") == 0 + or warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + } } } - } - }; + }; - $exit == 0 or do { - if ($recursiveFlag) { - warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; - } else { - warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; - } - }; + $exit == 0 or do { + if ($recursiveFlag) { + warn "CRITICAL ERROR: $zfs snapshot -r $snap failed, $?"; + } else { + warn "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; + } + }; + } } + if ($config{$dataset}{'post_snapshot_script'}) { if (!$presnapshotfailure or $config{$dataset}{'force_post_snapshot_script'}) { $ENV{'SANOID_TARGET'} = $dataset; - $ENV{'SANOID_SNAPNAME'} = $snapname; - $ENV{'SANOID_TYPE'} = $snapData->{type}; - $ENV{'SANOID_BATCH'} = $batch; + $ENV{'SANOID_TARGETS'} = $datasetString; + $ENV{'SANOID_SNAPNAME'} = @snapshots[0]; + $ENV{'SANOID_SNAPNAMES'} = $snapshotString; + $ENV{'SANOID_TYPES'} = $typeString; $ENV{'SANOID_SCRIPT'} = 'post'; $ENV{'SANOID_PRE_FAILURE'} = $presnapshotfailure; if ($args{'verbose'}) { print "executing post_snapshot_script '".$config{$dataset}{'post_snapshot_script'}."' on dataset '$dataset'\n"; } @@ -623,9 +643,10 @@ sub take_snapshots { } delete $ENV{'SANOID_TARGET'}; + delete $ENV{'SANOID_TARGETS'}; delete $ENV{'SANOID_SNAPNAME'}; - delete $ENV{'SANOID_TYPE'}; - delete $ENV{'SANOID_BATCH'}; + delete $ENV{'SANOID_SNAPNAMES'}; + delete $ENV{'SANOID_TYPES'}; delete $ENV{'SANOID_SCRIPT'}; delete $ENV{'SANOID_PRE_FAILURE'}; }