support resumable zfs send/receive
This commit is contained in:
parent
85cc99c9e6
commit
d5f4b5abba
127
syncoid
127
syncoid
|
@ -19,7 +19,7 @@ use Sys::Hostname;
|
||||||
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
|
my %args = ('sshkey' => '', 'sshport' => '', 'sshcipher' => '', 'sshoption' => [], 'target-bwlimit' => '', 'source-bwlimit' => '');
|
||||||
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
|
GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsnaps", "recursive|r",
|
||||||
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
|
"source-bwlimit=s", "target-bwlimit=s", "sshkey=s", "sshport=i", "sshcipher|c=s", "sshoption|o=s@",
|
||||||
"debug", "quiet", "no-stream", "no-sync-snap") or pod2usage(2);
|
"debug", "quiet", "no-stream", "no-sync-snap", "resume") or pod2usage(2);
|
||||||
|
|
||||||
my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
|
my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
|
||||||
|
|
||||||
|
@ -161,35 +161,55 @@ sub syncdataset {
|
||||||
# does the target filesystem exist yet?
|
# does the target filesystem exist yet?
|
||||||
my $targetexists = targetexists($targethost,$targetfs,$targetisroot);
|
my $targetexists = targetexists($targethost,$targetfs,$targetisroot);
|
||||||
|
|
||||||
# build hashes of the snaps on the source and target filesystems.
|
my $receiveextraargs = "";
|
||||||
|
my $receivetoken;
|
||||||
|
if (defined $args{'resume'}) {
|
||||||
|
# save state of interrupted receive stream
|
||||||
|
$receiveextraargs = "-s";
|
||||||
|
|
||||||
%snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot);
|
if ($targetexists) {
|
||||||
|
# check remote dataset for receive resume token (interrupted receive)
|
||||||
|
$receivetoken = getreceivetoken($targethost,$targetfs,$targetisroot);
|
||||||
|
|
||||||
if ($targetexists) {
|
if ($debug && defined($receivetoken)) {
|
||||||
my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot);
|
print "DEBUG: got receive resume token: $receivetoken: \n";
|
||||||
my %sourcesnaps = %snaps;
|
}
|
||||||
%snaps = (%sourcesnaps, %targetsnaps);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (defined $args{'dumpsnaps'}) {
|
|
||||||
print "merged snapshot list of $targetfs: \n";
|
|
||||||
dumphash(\%snaps);
|
|
||||||
print "\n\n\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
# create a new syncoid snapshot on the source filesystem.
|
|
||||||
my $newsyncsnap;
|
|
||||||
if (!defined $args{'no-sync-snap'}) {
|
|
||||||
$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
|
|
||||||
} else {
|
|
||||||
# we don't want sync snapshots created, so use the newest snapshot we can find.
|
|
||||||
$newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot);
|
|
||||||
if ($newsyncsnap eq 0) {
|
|
||||||
warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n";
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
my $newsyncsnap;
|
||||||
|
|
||||||
|
# skip snapshot checking/creation in case of resumed receive
|
||||||
|
if (!defined($receivetoken)) {
|
||||||
|
# build hashes of the snaps on the source and target filesystems.
|
||||||
|
|
||||||
|
%snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot);
|
||||||
|
|
||||||
|
if ($targetexists) {
|
||||||
|
my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot);
|
||||||
|
my %sourcesnaps = %snaps;
|
||||||
|
%snaps = (%sourcesnaps, %targetsnaps);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (defined $args{'dumpsnaps'}) {
|
||||||
|
print "merged snapshot list of $targetfs: \n";
|
||||||
|
dumphash(\%snaps);
|
||||||
|
print "\n\n\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!defined $args{'no-sync-snap'}) {
|
||||||
|
# create a new syncoid snapshot on the source filesystem.
|
||||||
|
$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
|
||||||
|
} else {
|
||||||
|
# we don't want sync snapshots created, so use the newest snapshot we can find.
|
||||||
|
$newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot);
|
||||||
|
if ($newsyncsnap eq 0) {
|
||||||
|
warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# there is currently (2014-09-01) a bug in ZFS on Linux
|
# there is currently (2014-09-01) a bug in ZFS on Linux
|
||||||
# that causes readonly to always show on if it's EVER
|
# that causes readonly to always show on if it's EVER
|
||||||
# been turned on... even when it's off... unless and
|
# been turned on... even when it's off... unless and
|
||||||
|
@ -222,7 +242,7 @@ sub syncdataset {
|
||||||
my $oldestsnapescaped = escapeshellparam($oldestsnap);
|
my $oldestsnapescaped = escapeshellparam($oldestsnap);
|
||||||
|
|
||||||
my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped";
|
my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefsescaped\@$oldestsnapescaped";
|
||||||
my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped";
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
|
|
||||||
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot);
|
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap",0,$sourceisroot);
|
||||||
my $disp_pvsize = readablebytes($pvsize);
|
my $disp_pvsize = readablebytes($pvsize);
|
||||||
|
@ -286,6 +306,27 @@ sub syncdataset {
|
||||||
# setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly);
|
# setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
# resume interrupted receive if there is a valid resume $token
|
||||||
|
# and because this will ony resume the receive to the next
|
||||||
|
# snapshot, do a normal sync after that
|
||||||
|
if (defined($receivetoken)) {
|
||||||
|
my $sendcmd = "$sourcesudocmd $zfscmd send -t $receivetoken";
|
||||||
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
|
my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken);
|
||||||
|
my $disp_pvsize = readablebytes($pvsize);
|
||||||
|
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
|
||||||
|
my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
|
||||||
|
|
||||||
|
if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; }
|
||||||
|
if ($debug) { print "DEBUG: $synccmd\n"; }
|
||||||
|
system("$synccmd") == 0
|
||||||
|
or die "CRITICAL ERROR: $synccmd failed: $?";
|
||||||
|
|
||||||
|
# a resumed transfer will only be done to the next snapshot,
|
||||||
|
# so do an normal sync cycle
|
||||||
|
return syncdataset($sourcehost, $sourcefs, $targethost, $targetfs);
|
||||||
|
}
|
||||||
|
|
||||||
# find most recent matching snapshot and do an -I
|
# find most recent matching snapshot and do an -I
|
||||||
# to the new snapshot
|
# to the new snapshot
|
||||||
|
|
||||||
|
@ -326,7 +367,7 @@ sub syncdataset {
|
||||||
}
|
}
|
||||||
|
|
||||||
my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap";
|
my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefsescaped\@$matchingsnapescaped $sourcefsescaped\@$newsyncsnap";
|
||||||
my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfsescaped";
|
my $recvcmd = "$targetsudocmd $zfscmd receive $receiveextraargs -F $targetfsescaped";
|
||||||
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
|
my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot);
|
||||||
my $disp_pvsize = readablebytes($pvsize);
|
my $disp_pvsize = readablebytes($pvsize);
|
||||||
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
|
if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; }
|
||||||
|
@ -931,7 +972,7 @@ sub getsnaps() {
|
||||||
|
|
||||||
|
|
||||||
sub getsendsize {
|
sub getsendsize {
|
||||||
my ($sourcehost,$snap1,$snap2,$isroot) = @_;
|
my ($sourcehost,$snap1,$snap2,$isroot,$receivetoken) = @_;
|
||||||
|
|
||||||
my $snap1escaped = escapeshellparam($snap1);
|
my $snap1escaped = escapeshellparam($snap1);
|
||||||
my $snap2escaped = escapeshellparam($snap2);
|
my $snap2escaped = escapeshellparam($snap2);
|
||||||
|
@ -957,6 +998,12 @@ sub getsendsize {
|
||||||
$snaps = "$snap1escaped";
|
$snaps = "$snap1escaped";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# in case of a resumed receive, get the remaining
|
||||||
|
# size based on the resume token
|
||||||
|
if (defined($receivetoken)) {
|
||||||
|
$snaps = "-t $receivetoken";
|
||||||
|
}
|
||||||
|
|
||||||
my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps";
|
my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps";
|
||||||
if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; }
|
if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; }
|
||||||
|
|
||||||
|
@ -969,7 +1016,13 @@ sub getsendsize {
|
||||||
# size of proposed xfer in bytes, but we need to remove
|
# size of proposed xfer in bytes, but we need to remove
|
||||||
# human-readable crap from it
|
# human-readable crap from it
|
||||||
my $sendsize = pop(@rawsize);
|
my $sendsize = pop(@rawsize);
|
||||||
$sendsize =~ s/^size\s*//;
|
# the output format is different in case of
|
||||||
|
# a resumed receive
|
||||||
|
if (defined($receivetoken)) {
|
||||||
|
$sendsize =~ s/.*\s([0-9]+)$/$1/;
|
||||||
|
} else {
|
||||||
|
$sendsize =~ s/^size\s*//;
|
||||||
|
}
|
||||||
chomp $sendsize;
|
chomp $sendsize;
|
||||||
|
|
||||||
# to avoid confusion with a zero size pv, give sendsize
|
# to avoid confusion with a zero size pv, give sendsize
|
||||||
|
@ -1006,6 +1059,21 @@ sub escapeshellparam {
|
||||||
return "'$par'";
|
return "'$par'";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub getreceivetoken() {
|
||||||
|
my ($rhost,$fs,$isroot) = @_;
|
||||||
|
my $token = getzfsvalue($rhost,$fs,$isroot,"receive_resume_token");
|
||||||
|
|
||||||
|
if ($token ne '-' && $token ne '') {
|
||||||
|
return $token;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($debug) {
|
||||||
|
print "DEBUG: no receive token found \n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
__END__
|
__END__
|
||||||
|
|
||||||
=head1 NAME
|
=head1 NAME
|
||||||
|
@ -1043,3 +1111,4 @@ Options:
|
||||||
--quiet Suppresses non-error output
|
--quiet Suppresses non-error output
|
||||||
--dumpsnaps Dumps a list of snapshots during the run
|
--dumpsnaps Dumps a list of snapshots during the run
|
||||||
--no-command-checks Do not check command existence before attempting transfer. Not recommended
|
--no-command-checks Do not check command existence before attempting transfer. Not recommended
|
||||||
|
--resume Save the state of unfinished receive streams and resume interrupted ones if available
|
||||||
|
|
Loading…
Reference in New Issue