diff --git a/CHANGELIST b/CHANGELIST index e7e9645..b920e93 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,10 @@ +1.4.8 added --no-stream argument to syncoid: allows use of -i incrementals (do not replicate a full snapshot stream, only a + direct incremental update from oldest to most recent snapshot) instead of the normal -I incrementals which include + all intermediate snapshots. + + added --no-sync-snap, which has syncoid replicate using only the newest PRE-EXISTING snapshot on source, + instead of default behavior in which syncoid creates a new, ephemeral syncoid snapshot. + 1.4.7a (syncoid only) added standard invocation output when called without source or target as per @rriley and @fajarnugraha suggestions diff --git a/VERSION b/VERSION index be05bba..b2e46d1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.7 +1.4.8 diff --git a/sanoid b/sanoid index 12806e0..2e7d490 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.7'; +my $version = '1.4.8'; use strict; use Config::IniFiles; # read samba-style conf file @@ -46,7 +46,7 @@ if ($args{'version'}) { print "INFO: Sanoid version: $version\n"; } if ($args{'cron'} || $args{'noargs'}) { if ($args{'noargs'}) { print "INFO: No arguments given - assuming --cron and --verbose.\n"; } - $args{'verbose'} = 1; + if (!$args{'quiet'}) { $args{'verbose'} = 1; } take_snapshots (@params); prune_snapshots (@params); } else { @@ -1032,8 +1032,8 @@ sub getargs { my %validargs; my %novalueargs; - push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','configdir'; - push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly'; + push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','configdir','quiet'; + push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','quiet'; foreach my $item (@validargs) { $validargs{$item}=1; } foreach my $item (@novalueargs) { $novalueargs{$item}=1; } diff --git a/sanoid.spec b/sanoid.spec index 7f7900c..97ac120 100644 --- a/sanoid.spec +++ b/sanoid.spec @@ -8,7 +8,6 @@ Group: Applications/System License: GPLv3 URL: https://github.com/jimsalterjrs/sanoid Source0: https://github.com/jimsalterjrs/sanoid/archive/sanoid-master.zip -Patch0: sanoid-syncoid-sshkey.patch #BuildRequires: Requires: perl diff --git a/syncoid b/syncoid index c9b31f2..9345fea 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.7a'; +my $version = '1.4.8'; use strict; use warnings; @@ -110,7 +110,7 @@ sub getchilddatasets { if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } - my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -Hr $fs |"; + my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -t filesystem,volume -Hr $fs |"; if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; } open FH, $getchildrencmd; my @children = ; @@ -135,10 +135,10 @@ sub syncdataset { # build hashes of the snaps on the source and target filesystems. - %snaps = getsnaps('source',$sourcehost,$sourcefs); + %snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot); if ($targetexists) { - my %targetsnaps = getsnaps('target',$targethost,$targetfs); + my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot); my %sourcesnaps = %snaps; %snaps = (%sourcesnaps, %targetsnaps); } @@ -146,7 +146,14 @@ sub syncdataset { if ($args{'dumpsnaps'}) { print "merged snapshot list: \n"; dumphash(\%snaps); print "\n\n\n"; } # create a new syncoid snapshot on the source filesystem. - my $newsyncsnap = newsyncsnap($sourcehost,$sourcefs); + my $newsyncsnap; + if (!defined ($args{'no-sync-snap'}) ) { + $newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot); + } else { + # we don't want sync snapshots created, so use the newest snapshot we can find. + $newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot); + if ($newsyncsnap eq 0) { die "CRITICAL: no snapshots exist on source, and you asked for --no-sync-snap.\n"; } + } # there is currently (2014-09-01) a bug in ZFS on Linux # that causes readonly to always show on if it's EVER @@ -161,13 +168,23 @@ sub syncdataset { if (! $targetexists) { # do an initial sync from the oldest source snapshot # THEN do an -I to the newest - if ($debug) { print "DEBUG: target $targetfs does not exist. Finding oldest available snapshot on source $sourcefs ...\n"; } + if ($debug) { + if (!defined ($args{'no-stream'}) ) { + print "DEBUG: target $targetfs does not exist. Finding oldest available snapshot on source $sourcefs ...\n"; + } else { + print "DEBUG: target $targetfs does not exist, and --no-stream selected. Finding newest available snapshot on source $sourcefs ...\n"; + } + } my $oldestsnap = getoldestsnapshot(\%snaps); if (! $oldestsnap) { # getoldestsnapshot() returned false, so use new sync snapshot if ($debug) { print "DEBUG: getoldestsnapshot() returned false, so using $newsyncsnap.\n"; } $oldestsnap = $newsyncsnap; } + + # if --no-stream is specified, our full needs to be the newest snapshot, not the oldest. + if (defined $args{'no-stream'}) { $oldestsnap = getnewestsnapshot(\%snaps); } + my $sendcmd = "$sourcesudocmd $zfscmd send $sourcefs\@$oldestsnap"; my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; @@ -175,7 +192,13 @@ sub syncdataset { my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); - if (!$quiet) { print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; } + if (!$quiet) { + if (!defined ($args{'no-stream'}) ) { + print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; + } else { + print "INFO: --no-stream selected; sending newest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; + } + } if ($debug) { print "DEBUG: $synccmd\n"; } # make sure target is (still) not currently in receive. @@ -186,8 +209,10 @@ sub syncdataset { or die "CRITICAL ERROR: $synccmd failed: $?"; # now do an -I to the new sync snapshot, assuming there were any snapshots - # other than the new sync snapshot to begin with, of course - if ($oldestsnap ne $newsyncsnap) { + # other than the new sync snapshot to begin with, of course - and that we + # aren't invoked with --no-stream, in which case a full of the newest snap + # available was all we needed to do + if (!defined ($args{'no-stream'}) && ($oldestsnap ne $newsyncsnap) ) { # get current readonly status of target, then set it to on during sync # dyking this functionality out for the time being due to buggy mount/unmount behavior @@ -195,7 +220,7 @@ sub syncdataset { # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); - $sendcmd = "$sourcesudocmd $zfscmd send -I $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; + $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -208,8 +233,13 @@ sub syncdataset { if (!$quiet) { print "INFO: Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } - system($synccmd) == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; + + if ($oldestsnap ne $newsyncsnap) { + system($synccmd) == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + } else { + if (!$quiet) { print "INFO: no incremental sync needed; $oldestsnap is already the newest available snapshot.\n"; } + } # restore original readonly value to target after sync complete # dyking this functionality out for the time being due to buggy mount/unmount behavior @@ -235,32 +265,37 @@ sub syncdataset { die "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; } - # rollback target to matchingsnap - if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } - if ($targethost ne '') { - if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + if ($matchingsnap eq $newsyncsnap) { + # barf some text but don't touch the filesystem + if (!$quiet) { print "INFO: no snapshots on source newer than $newsyncsnap on target. Nothing to do, not syncing.\n"; } } else { - if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } - system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + # rollback target to matchingsnap + if ($debug) { print "DEBUG: rolling back target to $targetfs\@$matchingsnap...\n"; } + if ($targethost ne '') { + if ($debug) { print "$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } + system ("$sshcmd $targethost $targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + } else { + if ($debug) { print "$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap\n"; } + system ("$targetsudocmd $zfscmd rollback -R $targetfs\@$matchingsnap"); + } + + my $sendcmd = "$sourcesudocmd $zfscmd send $args{'streamarg'} $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; + my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); + my $disp_pvsize = readablebytes($pvsize); + if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } + my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); + + if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } + if ($debug) { print "DEBUG: $synccmd\n"; } + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; + + # restore original readonly value to target after sync complete + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + #setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } - - my $sendcmd = "$sourcesudocmd $zfscmd send -I $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; - my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); - my $disp_pvsize = readablebytes($pvsize); - if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } - my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); - - if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } - if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd") == 0 - or die "CRITICAL ERROR: $synccmd failed: $?"; - - # restore original readonly value to target after sync complete - # dyking this functionality out for the time being due to buggy mount/unmount behavior - # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. - #setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } # prune obsolete sync snaps on source and target. @@ -276,9 +311,9 @@ sub getargs { my %novaluearg; my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet'); + push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet','no-stream','no-sync-snap'); foreach my $item (@validargs) { $validarg{$item} = 1; } - push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet'); + push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet','no-stream','no-sync-snap'); foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } while (my $rawarg = shift(@args)) { @@ -338,6 +373,8 @@ sub getargs { if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; } if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; } + if (defined $args{'no-stream'}) { $args{'streamarg'} = '-i'; } else { $args{'streamarg'} = '-I'; } + if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } if (!defined $args{'compress'}) { $args{'compress'} = 'default'; } @@ -569,6 +606,19 @@ sub getoldestsnapshot { return $snap; } # must not have had any snapshots on source - luckily, we already made one, amirite? + die "CRIT: getoldestsnapshot() could not find any snapshots on source!\n"; + return 0; +} + +sub getnewestsnapshot { + my $snaps = shift; + foreach my $snap ( sort { $snaps{'source'}{$b}{'ctime'}<=>$snaps{'source'}{$a}{'ctime'} } keys %{ $snaps{'source'} }) { + # return on first snap found - it's the newest + print "NEWEST SNAPSHOT: $snap\n"; + return $snap; + } + # must not have had any snapshots on source - looks like we'd better create one! + die "CRIT: getnewestsnapshot() could not find any snapshots on source!\n"; return 0; } @@ -763,7 +813,7 @@ sub getssh { if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); - open FH, "$sshcmd -M -S $socket -o ControlPersist=yes $sshport $rhost exit |"; + open FH, "$sshcmd -M -S $socket -o ControlPersist=1m $sshport $rhost exit |"; close FH; $rhost = "-S $socket $rhost"; } else { @@ -787,7 +837,7 @@ sub getsnaps() { if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } - my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 creation $fs |"; + my $getsnapcmd = "$rhost $mysudocmd $zfscmd get -Hpd 1 -t snapshot creation $fs |"; if ($debug) { print "DEBUG: getting list of snapshots on $fs using $getsnapcmd...\n"; } open FH, $getsnapcmd; my @rawsnaps = ; @@ -798,11 +848,9 @@ sub getsnaps() { if ($line =~ /$fs\@/) { chomp $line; my $ctime = $line; - $ctime =~ s/^.*creation\s*//; - $ctime =~ s/\s*-$//; + $ctime =~ s/^.*\screation\s*(\d*).*/$1/; my $snap = $line; - $snap =~ s/\s*creation.*$//; - $snap =~ s/^\S*\@//; + $snap =~ s/^\S*\@(\S*)\s*creation.*$/$1/; $snaps{$type}{$snap}{'ctime'}=$ctime; } } @@ -820,7 +868,7 @@ sub getsendsize { my $snaps; if ($snap2) { # if we got a $snap2 argument, we want an incremental send estimate from $snap1 to $snap2. - $snaps = "-I $snap1 $snap2"; + $snaps = "$args{'streamarg'} $snap1 $snap2"; } else { # if we didn't get a $snap2 arg, we want a full send estimate for $snap1. $snaps = "$snap1";