Merge cd0b9f362b into b31ed6e325

Merge pull request #916 from 0xFelix/zstdmt
syncoid: Add zstdmt compress options
2024-04-22 20:09:31 +02:00 · 2024-04-22 12:14:40 -04:00 · 2024-04-22 12:13:43 -04:00 · 2024-04-22 12:13:28 -04:00 · 2024-04-22 12:12:58 -04:00 · 2024-04-22 12:12:28 -04:00
15 changed files with 107 additions and 21 deletions
--- a/3
+++ b/3
@ -25,6 +25,9 @@ if ($args{'path'} eq '') {
 	}
 }

+# resolve given path to a canonical one
+$args{'path'} = Cwd::realpath($args{'path'});
+
 my $dataset = getdataset($args{'path'});

 my %versions = getversions($args{'path'}, $dataset);
--- a/103
+++ b/103
@ -498,7 +498,6 @@ sub syncdataset {

 		my $ret;
 		if (defined $origin) {
-			writelog('INFO', "Clone is recreated on target $targetfs based on $origin");
 			($ret, $stdout) = syncclone($sourcehost, $sourcefs, $origin, $targethost, $targetfs, $oldestsnap);
 			if ($ret) {
 				writelog('INFO', "clone creation failed, trying ordinary replication as fallback");
@ -506,12 +505,6 @@ sub syncdataset {
 				return 0;
 			}
 		} else {
-			if (!defined ($args{'no-stream'}) ) {
-				writelog('INFO', "Sending oldest full snapshot $sourcefs\@$oldestsnap to new target filesystem:");
-			} else {
-				writelog('INFO', "--no-stream selected; sending newest full snapshot $sourcefs\@$oldestsnap to new target filesystem:");
-			}
-
 			($ret, $stdout) = syncfull($sourcehost, $sourcefs, $targethost, $targetfs, $oldestsnap);
 		}

@ -532,8 +525,6 @@ sub syncdataset {
 			# $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly');
 			# setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on');

-			writelog('INFO', "Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap:");
-
 			(my $ret, $stdout) = syncincremental($sourcehost, $sourcefs, $targethost, $targetfs, $oldestsnap, $newsyncsnap, 0);

 			if ($ret != 0) {
@ -865,6 +856,16 @@ sub syncdataset {
 		# those that exist on the source. Remaining are the snapshots
 		# that are only on the target. Then sort to remove the oldest
 		# snapshots first.
+
+		# regather snapshots on source and target
+		%snaps = getsnaps('source',$sourcehost,$sourcefs,$sourceisroot,0);
+
+		if ($targetexists) {
+		    my %targetsnaps = getsnaps('target',$targethost,$targetfs,$targetisroot,0);
+		    my %sourcesnaps = %snaps;
+		    %snaps = (%sourcesnaps, %targetsnaps);
+		}
+
 		my @to_delete = sort { sortsnapshots(\%snaps, $a, $b) } grep {!exists $snaps{'source'}{$_}} keys %{ $snaps{'target'} };
 		while (@to_delete) {
 			# Create batch of snapshots to remove
@ -898,7 +899,6 @@ sub runsynccmd {
 	my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize);
 	my $sendoptions;
 	if ($sendsource =~ / -t /) {
-		writelog('INFO', "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):");
 		$sendoptions = getoptionsline(\@sendoptions, ('P','V','e','v'));
 	} elsif ($sendsource =~ /#/) {
 		$sendoptions = getoptionsline(\@sendoptions, ('L','V','c','e','w'));
@ -934,12 +934,13 @@ sub runsynccmd {
 	my $recvcmd = "$targetsudocmd $zfscmd receive $recvoptions $targetfsescaped 2>&1";

 	my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot);
-	writelog('INFO', "Sync size: ~$disp_pvsize");
+	writelog('DEBUG', "sync size: ~$disp_pvsize");
 	writelog('DEBUG', "$synccmd");

 	# make sure target is (still) not currently in receive.
 	if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
-		writelog('WARN', "Cannot sync now: $targetfs is already target of a zfs receive process.");
+		my $targetname = buildnicename($targethost, $targetfs);
+		writelog('WARN', "Cannot sync now: $targetname is already target of a zfs receive process.");
 		return (1, '');
 	}

@ -971,6 +972,16 @@ sub syncfull {
 	my $sendsource = "$sourcefsescaped\@$snapescaped";
 	my $pvsize = getsendsize($sourcehost,"$sourcefs\@$snapname",0,$sourceisroot);

+	my $srcname = buildnicename($sourcehost, $sourcefs, $snapname);
+	my $targetname = buildnicename($targethost, $targetfs);
+	my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize);
+
+	if (!defined ($args{'no-stream'}) ) {
+		writelog('INFO', "Sending oldest full snapshot $srcname to new target filesystem $targetname (~ $disp_pvsize):");
+	} else {
+		writelog('INFO', "--no-stream selected; sending newest full snapshot $srcname to new target filesystem $targetname: (~ $disp_pvsize)");
+	}
+
 	return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize);
 } # end syncfull()

@ -1011,8 +1022,11 @@ sub syncincremental {
 			foreach my $i (0..(scalar(@intsnaps) - 2)) {
 				my $snapa = $intsnaps[$i];
 				my $snapb = $intsnaps[$i + 1];
-				writelog('INFO', "Performing an incremental sync between '$snapa' and '$snapb'");
-				syncincremental($sourcehost, $sourcefs, $targethost, $targetfs, $snapa, $snapb, 1) == 0 or return $?;
+				(my $ret, my $stdout) = syncincremental($sourcehost, $sourcefs, $targethost, $targetfs, $snapa, $snapb, 1);
+
+				if ($ret != 0) {
+					return ($ret, $stdout);
+				}
 			}

 			# Return after finishing the -i syncs so that we don't try to do another -I
@ -1026,6 +1040,12 @@ sub syncincremental {
 	my $sendsource = "$streamarg $sourcefsescaped\@$fromsnapescaped $sourcefsescaped\@$tosnapescaped";
 	my $pvsize = getsendsize($sourcehost,"$sourcefs\@$fromsnap","$sourcefs\@$tosnap",$sourceisroot);

+	my $srcname = buildnicename($sourcehost, $sourcefs, $fromsnap);
+	my $targetname = buildnicename($targethost, $targetfs);
+	my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize);
+
+	writelog('INFO', "Sending incremental $srcname ... $tosnap to $targetname (~ $disp_pvsize):");
+
 	return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize);
 } # end syncincremental()

@ -1038,6 +1058,12 @@ sub syncclone {
 	my $sendsource = "-i $originescaped $sourcefsescaped\@$tosnapescaped";
 	my $pvsize = getsendsize($sourcehost,$origin,"$sourcefs\@$tosnap",$sourceisroot);

+	my $srcname = buildnicename($sourcehost, $origin);
+	my $targetname = buildnicename($targethost, $targetfs);
+	my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize);
+
+	writelog('INFO', "Clone is recreated on target $targetname based on $srcname (~ $disp_pvsize):");
+
 	return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize);
 } # end syncclone()

@ -1047,6 +1073,12 @@ sub syncresume {
 	my $sendsource = "-t $receivetoken";
 	my $pvsize = getsendsize($sourcehost,"","",$sourceisroot,$receivetoken);

+	my $srcname = buildnicename($sourcehost, $sourcefs);
+	my $targetname = buildnicename($targethost, $targetfs);
+	my $disp_pvsize = $pvsize == 0 ? 'UNKNOWN' : readablebytes($pvsize);
+
+	writelog('INFO', "Resuming interrupted zfs send/receive from $srcname to $targetname (~ $disp_pvsize remaining):");
+
 	return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, $pvsize);
 } # end syncresume()

@ -1058,6 +1090,11 @@ sub syncbookmark {
 	my $tosnapescaped = escapeshellparam($tosnap);
 	my $sendsource = "-i $sourcefsescaped#$bookmarkescaped $sourcefsescaped\@$tosnapescaped";

+	my $srcname = buildnicename($sourcehost, $sourcefs, '', $bookmark);
+	my $targetname = buildnicename($targethost, $targetfs);
+
+	writelog('INFO', "Sending incremental $srcname ... $tosnap to $targetname:");
+
 	return runsynccmd($sourcehost, $sourcefs, $sendsource, $targethost, $targetfs, 0);
 } # end syncbookmark

@ -1110,12 +1147,24 @@ sub compressargset {
 			decomrawcmd => 'zstd',
 			decomargs   => '-dc',
 		},
+		'zstdmt-fast' => {
+			rawcmd      => 'zstdmt',
+			args        => '-3',
+			decomrawcmd => 'zstdmt',
+			decomargs   => '-dc',
+		},
 		'zstd-slow' => {
 			rawcmd      => 'zstd',
 			args        => '-19',
 			decomrawcmd => 'zstd',
 			decomargs   => '-dc',
 		},
+		'zstdmt-slow' => {
+			rawcmd      => 'zstdmt',
+			args        => '-19',
+			decomrawcmd => 'zstdmt',
+			decomargs   => '-dc',
+		},
 		'xz' => {
 			rawcmd      => 'xz',
 			args        => '',
@ -1138,7 +1187,7 @@ sub compressargset {

 	if ($value eq 'default') {
 		$value = $DEFAULT_COMPRESSION;
-	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstd-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) {
+	} elsif (!(grep $value eq $_, ('gzip', 'pigz-fast', 'pigz-slow', 'zstd-fast', 'zstdmt-fast', 'zstd-slow', 'zstdmt-slow', 'lz4', 'xz', 'lzo', 'default', 'none'))) {
 		writelog('WARN', "Unrecognised compression value $value, defaulting to $DEFAULT_COMPRESSION");
 		$value = $DEFAULT_COMPRESSION;
 	}
@ -1507,7 +1556,7 @@ sub getnewestsnapshot {
 	my $snaps = shift;
 	foreach my $snap (sort { sortsnapshots($snaps, $b, $a) } keys %{ $snaps{'source'} }) {
 		# return on first snap found - it's the newest
-		writelog('INFO', "NEWEST SNAPSHOT: $snap");
+		writelog('DEBUG', "NEWEST SNAPSHOT: $snap");
 		return $snap;
 	}
 	# must not have had any snapshots on source - looks like we'd better create one!
@ -2233,6 +2282,26 @@ sub snapisincluded {
 	return 1;
 }

+sub buildnicename {
+	my ($host,$fs,$snapname,$bookmarkname) = @_;
+
+	my $name;
+	if ($host) {
+		$host =~ s/-S \/tmp\/syncoid[a-zA-Z0-9-@]+ //g;
+		$name = "$host:$fs";
+	} else {
+		$name = "$fs";
+	}
+
+	if ($snapname) {
+		$name = "$name\@$snapname";
+	} elsif ($bookmarkname) {
+		$name = "$name#$bookmarkname";
+	}
+
+	return $name;
+}
+
 __END__

 =head1 NAME
@ -2251,7 +2320,7 @@ syncoid - ZFS snapshot replication tool

 Options:

-  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstd-slow, lz4, xz, lzo (default) & none
+  --compress=FORMAT     Compresses data during transfer. Currently accepted options are gzip, pigz-fast, pigz-slow, zstd-fast, zstdmt-fast, zstd-slow, zstdmt-slow, lz4, xz, lzo (default) & none
  --identifier=EXTRA    Extra identifier which is included in the snapshot name. Can be used for replicating to multiple targets.
  --recursive|r         Also transfers child datasets
  --skip-parent         Skips syncing of the parent dataset. Does nothing without '--recursive' option.
--- a/tests/run-tests.sh
+++ b/tests/run-tests.sh
@ -17,8 +17,11 @@ for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g);
    cd "${test}"
    echo -n y | bash run.sh > "${LOGFILE}" 2>&1

-    if [ $? -eq 0 ]; then
+    ret=$?
+    if [ $ret -eq 0 ]; then
        echo "[PASS]"
+    elif [ $ret -eq 130 ]; then
+        echo "[SKIPPED]"
    else
        echo "[FAILED] (see ${LOGFILE})"
    fi
--- a/tests/syncoid/001_bookmark_replication_intermediate/run.sh
+++ b/tests/syncoid/001_bookmark_replication_intermediate/run.sh
--- a/tests/syncoid/002_bookmark_replication_no_intermediate/run.sh
+++ b/tests/syncoid/002_bookmark_replication_no_intermediate/run.sh
--- a/tests/syncoid/003_force_delete/run.sh
+++ b/tests/syncoid/003_force_delete/run.sh
--- a/tests/syncoid/004_bookmark_replication_edge_case/run.sh
+++ b/tests/syncoid/004_bookmark_replication_edge_case/run.sh
--- a/tests/syncoid/005_reset_resume_state/run.sh
+++ b/tests/syncoid/005_reset_resume_state/run.sh
@ -28,6 +28,8 @@ zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src

 dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200

+sleep 1
+
 ../../../syncoid --debug --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst &
 syncoid_pid=$!
 sleep 5
--- a/tests/syncoid/006_reset_resume_state2/run.sh
+++ b/tests/syncoid/006_reset_resume_state2/run.sh
@ -28,6 +28,8 @@ zfs create -o mountpoint="${MOUNT_TARGET}" "${POOL_NAME}"/src

 dd if=/dev/urandom of="${MOUNT_TARGET}"/big_file bs=1M count=200

+sleep 1
+
 zfs snapshot "${POOL_NAME}"/src@big
 ../../../syncoid --debug --no-sync-snap --compress=none --source-bwlimit=2m "${POOL_NAME}"/src "${POOL_NAME}"/dst &
 syncoid_pid=$!
--- a/tests/syncoid/007_preserve_recordsize/run.sh
+++ b/tests/syncoid/007_preserve_recordsize/run.sh
--- a/tests/syncoid/008_force_delete_snapshot/run.sh
+++ b/tests/syncoid/008_force_delete_snapshot/run.sh
--- a/tests/syncoid/009_preserve_properties/run.sh
+++ b/tests/syncoid/009_preserve_properties/run.sh
--- a/tests/syncoid/010_filter_snaps/run.sh
+++ b/tests/syncoid/010_filter_snaps/run.sh
--- a/tests/syncoid/011_sync_out-of-order_snapshots/run.sh
+++ b/tests/syncoid/011_sync_out-of-order_snapshots/run.sh
@ -7,9 +7,13 @@ set -e

 . ../../common/lib.sh

-POOL_IMAGE="/tmp/jimsalterjrs_sanoid_815.img"
+if [ -z "$ALLOW_INVASIVE_TESTS" ]; then
+  exit 130
+fi
+
+POOL_IMAGE="/tmp/syncoid-test-11.zpool"
 POOL_SIZE="64M"
-POOL_NAME="jimsalterjrs_sanoid_815"
+POOL_NAME="syncoid-test-11"

 truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"

--- a/tests/syncoid/run-tests.sh
+++ b/tests/syncoid/run-tests.sh
@ -17,8 +17,11 @@ for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g);
    cd "${test}"
    echo | bash run.sh > "${LOGFILE}" 2>&1

-    if [ $? -eq 0 ]; then
+    ret=$?
+    if [ $ret -eq 0 ]; then
        echo "[PASS]"
+    elif [ $ret -eq 130 ]; then
+        echo "[SKIPPED]"
    else
        echo "[FAILED] (see ${LOGFILE})"
    fi
Author	SHA1	Message	Date
Aaron Whitehouse	accd8ccbac	Merge `cd0b9f362b` into `b31ed6e325`	2024-04-22 20:09:31 +02:00
Jim Salter	b31ed6e325	Merge pull request #916 from 0xFelix/zstdmt syncoid: Add zstdmt compress options	2024-04-22 12:14:40 -04:00
Jim Salter	fa2c16d65a	Merge pull request #905 from phreaker0/findoid-relative-path [findoid] support relative paths	2024-04-22 12:13:43 -04:00
Jim Salter	1207ea0062	Merge pull request #904 from phreaker0/tests-restructure test adaptions	2024-04-22 12:13:28 -04:00
Jim Salter	d800e5e17d	Merge pull request #903 from spicyFajitas/regather_snapshots--delete-target-snaps_task fix(syncoid): regather $snaps on --delete-target-snapshots flag	2024-04-22 12:12:58 -04:00
Jim Salter	1ee6815e5e	Merge pull request #910 from phreaker0/improve-output added missing status information about what is done and provide more details	2024-04-22 12:12:28 -04:00
0xFelix	8b7d29d5a0	syncoid: Add zstdmt compress options Add the zstdmt-fast and zstdmt-slow compress options to allow use of multithreading when using zstd compression. Signed-off-by: 0xFelix <felix@matouschek.org>	2024-04-20 18:41:43 +02:00
Christoph Klaffl	b4c8e4b499	Merge branch 'master' into improve-output	2024-04-18 14:30:04 +02:00
Jim Salter	45b1ce9e5d	Merge pull request #911 from phreaker0/fix-error-handling handle error output for filtered replications	2024-04-18 08:25:30 -04:00
Christoph Klaffl	6c1e31e551	handle error output for filtered replications	2024-04-18 08:22:37 +02:00
Christoph Klaffl	eb4fe8a01c	added missing status information about what is done and provide more details	2024-04-18 07:42:47 +02:00
Christoph Klaffl	d7ed4bdf54	support relative paths	2024-04-05 15:24:42 +02:00
Christoph Klaffl	4e86733c1a	missed debug statement	2024-04-05 15:22:13 +02:00
Christoph Klaffl	7c8a34eceb	* proper order of tests * timing fixes for fast NVME pools * skip invasive tests by default	2024-04-05 15:20:28 +02:00
Adam Fulton	d08b2882b7	finish rebase to master	2024-04-01 13:16:16 -05:00
Adam Fulton	f89372967f	fix(syncoid): regather $snaps on --delete-target-snapshots flag	2024-04-01 13:12:59 -05:00