From 99dcf7f3406b64f7f6ae261b41666fd2bbbaeec9 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:52:36 +0200 Subject: [PATCH 1/2] implemented support for zfs bookmarks, if no matching snapshots are found bookmarks are tried as fallback, both stream and no-stream cases are supported --- syncoid | 180 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 148 insertions(+), 32 deletions(-) diff --git a/syncoid b/syncoid index 6c569b2..3cbff6b 100755 --- a/syncoid +++ b/syncoid @@ -370,11 +370,48 @@ sub syncdataset { my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, $targetsize, \%snaps); + my $bookmark = 0; + my $bookmarkcreation = 0; + + my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps); if (! $matchingsnap) { - # no matching snapshot; we whined piteously already, but let's go ahead and return false - # now in case more child datasets need replication. - return 0; + # no matching snapshots, check for bookmarks as fallback + my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot); + + # check for matching guid of source bookmark and target snapshot (oldest first) + foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) { + my $guid = $snaps{'target'}{$snap}{'guid'}; + + if (defined $bookmarks{$guid}) { + # found a match + $bookmark = $bookmarks{$guid}{'name'}; + $bookmarkcreation = $bookmarks{$guid}{'creation'}; + $matchingsnap = $snap; + last; + } + } + + if (! $bookmark) { + # if we got this far, we failed to find a matching snapshot/bookmark. + + print "\n"; + print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; + print " Replication to target would require destroying existing\n"; + print " target. Cowardly refusing to destroy your existing target.\n\n"; + + # experience tells me we need a mollyguard for people who try to + # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... + + if ( $targetsize < (64*1024*1024) ) { + print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; + print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; + print " replication must be to a NON EXISTENT DATASET, which will\n"; + print " then be CREATED BY the initial replication process.\n\n"; + } + + # return false now in case more child datasets need replication. + return 0; + } } # make sure target is (still) not currently in receive. @@ -398,17 +435,68 @@ sub syncdataset { system ("$targetsudocmd $zfscmd rollback -R $targetfsescaped\@$matchingsnapescaped"); } - my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; - my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); - my $disp_pvsize = readablebytes($pvsize); - if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } - my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + my $nextsnapshot = 0; - if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } - if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + if ($bookmark) { + my $bookmarkescaped = escapeshellparam($bookmark); + + if (!defined $args{'no-stream'}) { + # if intermediate snapshots are needed we need to find the next oldest snapshot, + # do an replication to it and replicate as always from oldest to newest + # because bookmark sends doesn't support intermediates directly + foreach my $snap ( sort { $snaps{'source'}{$a}{'creation'}<=>$snaps{'source'}{$b}{'creation'} } keys %{ $snaps{'source'} }) { + if ($snaps{'source'}{$snap}{'creation'} >= $bookmarkcreation) { + $nextsnapshot = $snap; + last; + } + } + } + + # bookmark stream size can't be determined + my $pvsize = 0; + my $disp_pvsize = "UNKNOWN"; + + if ($nextsnapshot) { + my $nextsnapshotescaped = escapeshellparam($nextsnapshot); + + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$nextsnapshotescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + $matchingsnap = $nextsnapshot; + $matchingsnapescaped = escapeshellparam($matchingsnap); + } else { + my $sendcmd = "$sourcesudocmd $zfscmd send -i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } + } + + # do a normal replication if bookmarks aren't used or if previous + # bookmark replication was only done to the next oldest snapshot + if (!$bookmark || $nextsnapshot) { + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnapescaped"; + my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped"; + my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior @@ -900,31 +988,15 @@ sub pruneoldsyncsnaps { } sub getmatchingsnapshot { - my ($sourcefs, $targetfs, $targetsize, $snaps) = @_; + my ($sourcefs, $targetfs, $snaps) = @_; foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) { - if (defined $snaps{'target'}{$snap}{'guid'}) { + if (defined $snaps{'target'}{$snap}) { if ($snaps{'source'}{$snap}{'guid'} == $snaps{'target'}{$snap}{'guid'}) { return $snap; } } } - # if we got this far, we failed to find a matching snapshot. - - print "\n"; - print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n"; - print " Replication to target would require destroying existing\n"; - print " target. Cowardly refusing to destroy your existing target.\n\n"; - - # experience tells me we need a mollyguard for people who try to - # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... - - if ( $targetsize < (64*1024*1024) ) { - print " NOTE: Target $targetfs dataset is < 64MB used - did you mistakenly run\n"; - print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; - print " replication must be to a NON EXISTENT DATASET, which will\n"; - print " then be CREATED BY the initial replication process.\n\n"; - } return 0; } @@ -1049,6 +1121,50 @@ sub getsnaps() { return %snaps; } +sub getbookmarks() { + my ($rhost,$fs,$isroot,%bookmarks) = @_; + my $mysudocmd; + my $fsescaped = escapeshellparam($fs); + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + my $getbookmarkcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t bookmark guid,creation $fsescaped |"; + if ($debug) { print "DEBUG: getting list of bookmarks on $fs using $getbookmarkcmd...\n"; } + open FH, $getbookmarkcmd; + my @rawbookmarks = ; + close FH; + + # this is a little obnoxious. get guid,creation returns guid,creation on two separate lines + # as though each were an entirely separate get command. + + my $lastguid; + + foreach my $line (@rawbookmarks) { + # only import bookmark guids, creation from the specified filesystem + if ($line =~ /\Q$fs\E\#.*guid/) { + chomp $line; + $lastguid = $line; + $lastguid =~ s/^.*\tguid\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tguid.*$/$1/; + $bookmarks{$lastguid}{'name'}=$bookmark; + } elsif ($line =~ /\Q$fs\E\#.*creation/) { + chomp $line; + my $creation = $line; + $creation =~ s/^.*\tcreation\t*(\d*).*/$1/; + my $bookmark = $line; + $bookmark =~ s/^.*\#(.*)\tcreation.*$/$1/; + $bookmarks{$lastguid}{'creation'}=$creation; + } + } + + return %bookmarks; +} sub getsendsize { my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_; From a1d9e79e70548b88e4f702d3790d9f5e7c028ffc Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Fri, 27 Jul 2018 22:58:42 +0200 Subject: [PATCH 2/2] added two tests for zfs bookmark replication --- .../run.sh | 56 +++++++++++++++++++ .../run.sh | 56 +++++++++++++++++++ tests/syncoid/run-tests.sh | 27 +++++++++ 3 files changed, 139 insertions(+) create mode 100755 tests/syncoid/1_bookmark_replication_intermediate/run.sh create mode 100755 tests/syncoid/2_bookmark_replication_no_intermediate/run.sh create mode 100755 tests/syncoid/run-tests.sh diff --git a/tests/syncoid/1_bookmark_replication_intermediate/run.sh b/tests/syncoid/1_bookmark_replication_intermediate/run.sh new file mode 100755 index 0000000..11edb04 --- /dev/null +++ b/tests/syncoid/1_bookmark_replication_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-1.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-1" +TARGET_CHECKSUM="a23564d5bb8a2babc3ac8936fd82825ad9fff9c82d4924f5924398106bbda9f0 -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | grep -v syncoid_ | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh new file mode 100755 index 0000000..94ac690 --- /dev/null +++ b/tests/syncoid/2_bookmark_replication_no_intermediate/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# test replication with fallback to bookmarks and all intermediate snapshots + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-2.zpool" +POOL_SIZE="200M" +POOL_NAME="syncoid-test-2" +TARGET_CHECKSUM="2460d4d4417793d2c7a5c72cbea4a8a584c0064bf48d8b6daa8ba55076cba66d -" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create "${POOL_NAME}"/src +zfs snapshot "${POOL_NAME}"/src@snap1 +zfs bookmark "${POOL_NAME}"/src@snap1 "${POOL_NAME}"/src#snap1 +# initial replication +../../../syncoid --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst +# destroy last common snapshot on source +zfs destroy "${POOL_NAME}"/src@snap1 + +# create intermediate snapshots +# sleep is needed so creation time can be used for proper sorting +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap2 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap3 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap4 +sleep 1 +zfs snapshot "${POOL_NAME}"/src@snap5 + +# replicate which should fallback to bookmarks +../../../syncoid --no-stream --no-sync-snap --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst || exit 1 + +# verify +output=$(zfs list -t snapshot -r "${POOL_NAME}" -H -o name) +checksum=$(echo "${output}" | sha256sum) + +if [ "${checksum}" != "${TARGET_CHECKSUM}" ]; then + exit 1 +fi + +exit 0 diff --git a/tests/syncoid/run-tests.sh b/tests/syncoid/run-tests.sh new file mode 100755 index 0000000..a9843a5 --- /dev/null +++ b/tests/syncoid/run-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# run's all the available tests + +for test in */; do + if [ ! -x "${test}/run.sh" ]; then + continue + fi + + testName="${test%/}" + + LOGFILE=/tmp/syncoid_test_run_"${testName}".log + + pushd . > /dev/null + + echo -n "Running test ${testName} ... " + cd "${test}" + echo | bash run.sh > "${LOGFILE}" 2>&1 + + if [ $? -eq 0 ]; then + echo "[PASS]" + else + echo "[FAILED] (see ${LOGFILE})" + fi + + popd > /dev/null +done