From e963dd228fd565922a9e1971846854c989283f8e Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:05:01 +0000 Subject: [PATCH 01/50] Add tests README with instructions on testing --- tests/README.md | 30 ++++++++++++++++++++++++++++++ tests/run-tests.sh | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..65e5658 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,30 @@ +### Requirements ### +Tests must be run inside a virtual machine. This is for your own safety, as the tests may create and destroy zpools etc. + +A VM with 35GB of storage and 8 cores completes the tests in about 5 hours. + +#### Packages #### +The tests require the following packages to be installed in the VM (Ubuntu 20.04 package names are used, translate as appropriate): +``` +zfsutils-linux +libconfig-inifiles-perl +libcapture-tiny-perl +``` +``` +apt install zfsutils-linux libconfig-inifiles-perl libcapture-tiny-perl +``` + +#### Install sanoid within the VM #### +Install sanoid within the VM, for example +``` +apt install git +git clone https://github.com/jimsalterjrs/sanoid.git +mkdir /etc/sanoid/ +cp sanoid/sanoid.defaults.conf /etc/sanoid/ +``` + +### Run the tests ## +``` +cd sanoid/tests/ +./run-tests.sh +``` \ No newline at end of file diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 38054b0..9a2a080 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -# run's all the available tests +# runs all the available tests for test in */; do if [ ! -x "${test}/run.sh" ]; then From cdbc1915ba2f86143defee1396b89c8e35ffc866 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:27:04 +0000 Subject: [PATCH 02/50] Add new monitoring test scaffolding. --- tests/0_monitoring_tests/run.sh | 39 +++++++++++++++++++++++++++++++++ tests/README.md | 7 ++++++ 2 files changed, 46 insertions(+) create mode 100755 tests/0_monitoring_tests/run.sh diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh new file mode 100755 index 0000000..459751e --- /dev/null +++ b/tests/0_monitoring_tests/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -x + +# this test will take hourly, daily and monthly snapshots +# for the whole year of 2017 in the timezone Europe/Vienna +# sanoid is run hourly and no snapshots are pruned + +. ../common/lib.sh + +POOL_NAME="sanoid-test-1" +POOL_TARGET="" # root + +# UTC timestamp of start and end +START="1483225200" +END="1514761199" + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +timestamp=$START + +mkdir -p "${POOL_TARGET}" +truncate -s 5120M "${POOL_TARGET}"/zpool.img + +zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + + diff --git a/tests/README.md b/tests/README.md index 65e5658..33be8f6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,4 +27,11 @@ cp sanoid/sanoid.defaults.conf /etc/sanoid/ ``` cd sanoid/tests/ ./run-tests.sh +``` + +### Example using LXD VMs ### +``` +VM_NAME=focal-sanoid-test +lxc init ubuntu:focal $VM_NAME --vm -c limits.cpu=8 -c limits.memory=10GB + ``` \ No newline at end of file From e26671c97dd48ca45f851f5b737de6c2067747d3 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:28:10 +0000 Subject: [PATCH 03/50] Add monitoring test framework --- tests/0_monitoring_tests/sanoid.conf | 10 ++++++++++ tests/0_monitoring_tests/test_monitoring.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/0_monitoring_tests/sanoid.conf create mode 100644 tests/0_monitoring_tests/test_monitoring.py diff --git a/tests/0_monitoring_tests/sanoid.conf b/tests/0_monitoring_tests/sanoid.conf new file mode 100644 index 0000000..f5692f0 --- /dev/null +++ b/tests/0_monitoring_tests/sanoid.conf @@ -0,0 +1,10 @@ +[sanoid-test-1] + use_template = production + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py new file mode 100644 index 0000000..9398626 --- /dev/null +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved +# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this +# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. + + +import unittest + +class TestNothing(unittest.TestCase): + def test_nothing(self): + """Test""" + + # Test sanoid_snapshots_exist + self.assertEqual(1,1) + + +if __name__ == '__main__': + unittest.main() From 372eadcb3725d9c3ce0149d14ad931ba0fee1359 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:35:13 +0000 Subject: [PATCH 04/50] Add python test --- tests/0_monitoring_tests/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh index 459751e..5fa577d 100755 --- a/tests/0_monitoring_tests/run.sh +++ b/tests/0_monitoring_tests/run.sh @@ -36,4 +36,4 @@ function cleanUp { # export pool in any case trap cleanUp EXIT - +python3 test_monitoring.py From b3ee98a234b3eaacfff3ee34ab370c331cb66ba6 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:37:43 +0000 Subject: [PATCH 05/50] Test failure. --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 9398626..836f656 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -12,7 +12,7 @@ class TestNothing(unittest.TestCase): """Test""" # Test sanoid_snapshots_exist - self.assertEqual(1,1) + self.assertEqual(1,2) if __name__ == '__main__': From 3655c0277574e31c8c5d7167ada1a4544ef657e5 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:54:48 +0000 Subject: [PATCH 06/50] Test output with no zpool. --- tests/0_monitoring_tests/run.sh | 20 ++++++++++---------- tests/0_monitoring_tests/test_monitoring.py | 20 +++++++++++++++----- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh index 5fa577d..b28b0af 100755 --- a/tests/0_monitoring_tests/run.sh +++ b/tests/0_monitoring_tests/run.sh @@ -10,24 +10,24 @@ set -x POOL_NAME="sanoid-test-1" POOL_TARGET="" # root -# UTC timestamp of start and end -START="1483225200" -END="1514761199" +# # UTC timestamp of start and end +# START="1483225200" +# END="1514761199" # prepare setup checkEnvironment -disableTimeSync +# disableTimeSync -# set timezone -ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime +# # set timezone +# ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime -timestamp=$START +# timestamp=$START -mkdir -p "${POOL_TARGET}" -truncate -s 5120M "${POOL_TARGET}"/zpool.img +# mkdir -p "${POOL_TARGET}" +# truncate -s 5120M "${POOL_TARGET}"/zpool.img -zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img +# zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img function cleanUp { zpool export "${POOL_NAME}" diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 836f656..7810c1f 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -6,13 +6,23 @@ import unittest +import os -class TestNothing(unittest.TestCase): - def test_nothing(self): - """Test""" +sanoid_cmd = os.environ.get("SANOID") - # Test sanoid_snapshots_exist - self.assertEqual(1,2) +def monitor_snapshots_command(): + """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--monitor-metrics"], check=True, capture_output=True) + return return_info + + + +class TestMonitoringOutput(unittest.TestCase): + def test_no_zpool(self): + """Test what happens if there is no zpool at all""" + + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, "chicken") if __name__ == '__main__': From 1bdb60715cb4854ad33fbc88a11c5eb3505252e6 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:57:00 +0000 Subject: [PATCH 07/50] add subprocess import --- tests/0_monitoring_tests/test_monitoring.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 7810c1f..df9c163 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -5,8 +5,10 @@ # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -import unittest import os +import subprocess +import unittest + sanoid_cmd = os.environ.get("SANOID") From 123e35e80424c9765af0d99f402fa67e2e9d42fc Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 10:59:44 +0000 Subject: [PATCH 08/50] Fix --monitor-snapshots command in test --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index df9c163..fcab67a 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -14,7 +14,7 @@ sanoid_cmd = os.environ.get("SANOID") def monitor_snapshots_command(): """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" - return_info = subprocess.run([sanoid_cmd, "--monitor-metrics"], check=True, capture_output=True) + return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], check=True, capture_output=True) return return_info From c21d8d0168bc367cc49367bb1600863483b7b74e Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 11:03:29 +0000 Subject: [PATCH 09/50] Fix path to sanoid snapshot cache file --- tests/common/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 904c98f..89d1238 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -10,7 +10,7 @@ function setup { export SANOID="../../sanoid" # make sure that there is no cache file - rm -f /var/cache/sanoidsnapshots.txt + rm -f /var/cache/sanoid/snapshots.txt # install needed sanoid configuration files [ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf From 8bb8d38cc84763e5bb73c7d6c6368f9b78859042 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 11:07:49 +0000 Subject: [PATCH 10/50] Remove check on test --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index fcab67a..b1a7ebc 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -14,7 +14,7 @@ sanoid_cmd = os.environ.get("SANOID") def monitor_snapshots_command(): """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" - return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], check=True, capture_output=True) + return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) return return_info From c488df59835099a5c49429752ab6fff61dc62d9d Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 11:10:49 +0000 Subject: [PATCH 11/50] Fix test output. --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index b1a7ebc..8b37f3c 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -24,7 +24,7 @@ class TestMonitoringOutput(unittest.TestCase): """Test what happens if there is no zpool at all""" return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, "chicken") + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!\n") if __name__ == '__main__': From f93985111f4a8c5820c0733dd14a6a52eedd007d Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 14:19:05 +0000 Subject: [PATCH 12/50] add second section --- tests/0_monitoring_tests/sanoid.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/0_monitoring_tests/sanoid.conf b/tests/0_monitoring_tests/sanoid.conf index f5692f0..9297d5d 100644 --- a/tests/0_monitoring_tests/sanoid.conf +++ b/tests/0_monitoring_tests/sanoid.conf @@ -1,6 +1,9 @@ [sanoid-test-1] use_template = production +[sanoid-test-2] + use_template = demo + [template_production] hourly = 36 daily = 30 @@ -8,3 +11,6 @@ yearly = 0 autosnap = yes autoprune = no + +[template_demo] + daily = 60 \ No newline at end of file From 26777c2c68d7e29e1109523524a3774daf955388 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 14:23:18 +0000 Subject: [PATCH 13/50] Add 2nd section to sanoid.conf for monitoring test --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 8b37f3c..d7d9cbc 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -24,7 +24,7 @@ class TestMonitoringOutput(unittest.TestCase): """Test what happens if there is no zpool at all""" return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!\n") + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") if __name__ == '__main__': From 501c9921c447627ea00dae172dcd55a6dfb89a7a Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 14:34:08 +0000 Subject: [PATCH 14/50] Test return code for monitoring no pool --- tests/0_monitoring_tests/test_monitoring.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index d7d9cbc..22fb2da 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -19,12 +19,20 @@ def monitor_snapshots_command(): + class TestMonitoringOutput(unittest.TestCase): def test_no_zpool(self): """Test what happens if there is no zpool at all""" return_info = monitor_snapshots_command() self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 0) + + # def test_with_zpool_no_snapshots(self): + # """Test what happens if there is a zpool at all""" + + # return_info = monitor_snapshots_command() + # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") if __name__ == '__main__': From c8f2b3f28374380dbfcb082994b65c9d35846a1a Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:25:22 +0000 Subject: [PATCH 15/50] Fix return code of first test --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 22fb2da..f374354 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -26,7 +26,7 @@ class TestMonitoringOutput(unittest.TestCase): return_info = monitor_snapshots_command() self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") - self.assertEqual(return_info.returncode, 0) + self.assertEqual(return_info.returncode, 2) # def test_with_zpool_no_snapshots(self): # """Test what happens if there is a zpool at all""" From e556495deaa4f6214bb09c5d7658152bb65720c9 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:45:49 +0000 Subject: [PATCH 16/50] Create zpool, test. --- tests/0_monitoring_tests/run.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh index b28b0af..a6da1f5 100755 --- a/tests/0_monitoring_tests/run.sh +++ b/tests/0_monitoring_tests/run.sh @@ -24,11 +24,6 @@ checkEnvironment # timestamp=$START -# mkdir -p "${POOL_TARGET}" -# truncate -s 5120M "${POOL_TARGET}"/zpool.img - -# zpool create -f "${POOL_NAME}" "${POOL_TARGET}"/zpool.img - function cleanUp { zpool export "${POOL_NAME}" } From 37bce97d6e903d1217eb1741f1640c0676c92885 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:46:07 +0000 Subject: [PATCH 17/50] add test --- tests/0_monitoring_tests/test_monitoring.py | 28 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index f374354..1ef51d6 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -7,6 +7,7 @@ import os import subprocess +from tabnanny import check import unittest @@ -17,7 +18,10 @@ def monitor_snapshots_command(): return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) return return_info - +def run_sanoid_cron_command(): + """Runs sanoid and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) + return return_info class TestMonitoringOutput(unittest.TestCase): @@ -28,8 +32,26 @@ class TestMonitoringOutput(unittest.TestCase): self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") self.assertEqual(return_info.returncode, 2) - # def test_with_zpool_no_snapshots(self): - # """Test what happens if there is a zpool at all""" + def test_with_zpool_no_snapshots(self): + """Test what happens if there is a zpool, but with no snapshots""" + + # Make the zpool + if os.environ.get("POOL_TARGET") != "": + subprocess.run(["mkdir", "-p", os.environ.get("POOL_TARGET")], check=True) + + subprocess.run(["truncate", "-s", "5120M", os.environ.get("POOL_TARGET") + "/zpool.img"], check=True) + subprocess.run(["zpool", "create", "-f", os.environ.get("POOL_NAME"), os.environ.get("POOL_TARGET") + "/zpool.img"], check=True) + + # Run sanoid --monitor-snapshots before doing anything else + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 2) + + # Run sanoid and test again + # run_sanoid_cron_command() + # return_info = monitor_snapshots_command() + # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + # self.assertEqual(return_info.returncode, 2) # return_info = monitor_snapshots_command() # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") From 890c785d0b482bf53aa9cd14c2dbaad6310c0732 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:51:29 +0000 Subject: [PATCH 18/50] Fixes to zpool creation test. --- tests/0_monitoring_tests/test_monitoring.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 1ef51d6..3ee3896 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -36,11 +36,14 @@ class TestMonitoringOutput(unittest.TestCase): """Test what happens if there is a zpool, but with no snapshots""" # Make the zpool - if os.environ.get("POOL_TARGET") != "": + if os.environ.get("POOL_TARGET") == "": + pool_disk_image = "/zpool.img" + else: subprocess.run(["mkdir", "-p", os.environ.get("POOL_TARGET")], check=True) + pool_disk_image = os.environ.get("POOL_TARGET") + "/zpool.img" - subprocess.run(["truncate", "-s", "5120M", os.environ.get("POOL_TARGET") + "/zpool.img"], check=True) - subprocess.run(["zpool", "create", "-f", os.environ.get("POOL_NAME"), os.environ.get("POOL_TARGET") + "/zpool.img"], check=True) + subprocess.run(["truncate", "-s", "5120M", pool_disk_image], check=True) + subprocess.run(["zpool", "create", "-f", os.environ.get("POOL_NAME"), pool_disk_image], check=True) # Run sanoid --monitor-snapshots before doing anything else return_info = monitor_snapshots_command() From 3e9730d135b6900009980133ceb66bc9fdabf2a1 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:54:36 +0000 Subject: [PATCH 19/50] debugging --- tests/0_monitoring_tests/test_monitoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 3ee3896..4b83439 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -39,6 +39,7 @@ class TestMonitoringOutput(unittest.TestCase): if os.environ.get("POOL_TARGET") == "": pool_disk_image = "/zpool.img" else: + print(os.environ.get("POOL_TARGET")) subprocess.run(["mkdir", "-p", os.environ.get("POOL_TARGET")], check=True) pool_disk_image = os.environ.get("POOL_TARGET") + "/zpool.img" From 8571510685fa46252639615fdaf2166cae8d5f48 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 21:57:40 +0000 Subject: [PATCH 20/50] Fix pool_target issues in test. --- tests/0_monitoring_tests/test_monitoring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 4b83439..e516b11 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -36,10 +36,9 @@ class TestMonitoringOutput(unittest.TestCase): """Test what happens if there is a zpool, but with no snapshots""" # Make the zpool - if os.environ.get("POOL_TARGET") == "": + if not os.environ.get("POOL_TARGET"): pool_disk_image = "/zpool.img" else: - print(os.environ.get("POOL_TARGET")) subprocess.run(["mkdir", "-p", os.environ.get("POOL_TARGET")], check=True) pool_disk_image = os.environ.get("POOL_TARGET") + "/zpool.img" From 33648050dde0edd18616a574f932c2ed2e794523 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:13:13 +0000 Subject: [PATCH 21/50] Clean up monitoring tests zpool creation/teardown --- tests/0_monitoring_tests/test_monitoring.py | 31 ++++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index e516b11..1f221f0 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -12,6 +12,11 @@ import unittest sanoid_cmd = os.environ.get("SANOID") +pool_disk_image1 = "/zpool1.img" +pool_name1 = "sanoid-test-1" +pool_disk_image2 = "/zpool2.img" +pool_name2 = "sanoid-test-2" + def monitor_snapshots_command(): """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" @@ -32,19 +37,25 @@ class TestMonitoringOutput(unittest.TestCase): self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") self.assertEqual(return_info.returncode, 2) +class TestsWithZpool(unittest.TestCase): + """Tests that require a test zpool""" + + def setUp(self): + """Set up the zpool""" + subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True) + subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True) + + subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) + subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) + + def tearDown(self): + """Clean up on either passed or failed tests""" + subprocess.run(["zpool", "export", pool_name1]) + subprocess.run(["zpool", "export", pool_name2]) + def test_with_zpool_no_snapshots(self): """Test what happens if there is a zpool, but with no snapshots""" - # Make the zpool - if not os.environ.get("POOL_TARGET"): - pool_disk_image = "/zpool.img" - else: - subprocess.run(["mkdir", "-p", os.environ.get("POOL_TARGET")], check=True) - pool_disk_image = os.environ.get("POOL_TARGET") + "/zpool.img" - - subprocess.run(["truncate", "-s", "5120M", pool_disk_image], check=True) - subprocess.run(["zpool", "create", "-f", os.environ.get("POOL_NAME"), pool_disk_image], check=True) - # Run sanoid --monitor-snapshots before doing anything else return_info = monitor_snapshots_command() self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") From 49589e2a17e93234055faba46d9575d01e65ce6c Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:17:17 +0000 Subject: [PATCH 22/50] Comment out zpool export, as now in python --- tests/0_monitoring_tests/run.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh index a6da1f5..ec7fea1 100755 --- a/tests/0_monitoring_tests/run.sh +++ b/tests/0_monitoring_tests/run.sh @@ -7,8 +7,8 @@ set -x . ../common/lib.sh -POOL_NAME="sanoid-test-1" -POOL_TARGET="" # root +# POOL_NAME="sanoid-test-1" +# POOL_TARGET="" # root # # UTC timestamp of start and end # START="1483225200" @@ -24,11 +24,11 @@ checkEnvironment # timestamp=$START -function cleanUp { - zpool export "${POOL_NAME}" -} +# function cleanUp { +# zpool export "${POOL_NAME}" +# } # export pool in any case -trap cleanUp EXIT +# trap cleanUp EXIT python3 test_monitoring.py From 867b755bb1ae517ae4960f5954e21831f010628c Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:19:36 +0000 Subject: [PATCH 23/50] Delete temp disk images as part of teardown --- tests/0_monitoring_tests/test_monitoring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 1f221f0..76573a7 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -51,7 +51,9 @@ class TestsWithZpool(unittest.TestCase): def tearDown(self): """Clean up on either passed or failed tests""" subprocess.run(["zpool", "export", pool_name1]) + subprocess.run(["rm", "export", pool_disk_image1]) subprocess.run(["zpool", "export", pool_name2]) + subprocess.run(["rm", "export", pool_disk_image2]) def test_with_zpool_no_snapshots(self): """Test what happens if there is a zpool, but with no snapshots""" From 02aedfaaa210251a85ba232da609fb96a34aaa8e Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:22:39 +0000 Subject: [PATCH 24/50] Test monitoring after sanoid cron --- tests/0_monitoring_tests/test_monitoring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 76573a7..863a5b1 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -64,10 +64,10 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.returncode, 2) # Run sanoid and test again - # run_sanoid_cron_command() - # return_info = monitor_snapshots_command() - # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") - # self.assertEqual(return_info.returncode, 2) + run_sanoid_cron_command() + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 2) # return_info = monitor_snapshots_command() # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") From d0f89c96bb45b82f1e8eff63ce7c65cb79956985 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:31:15 +0000 Subject: [PATCH 25/50] Fix monitoring test after sanoid run --- tests/0_monitoring_tests/test_monitoring.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 863a5b1..47263e3 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -66,12 +66,9 @@ class TestsWithZpool(unittest.TestCase): # Run sanoid and test again run_sanoid_cron_command() return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") self.assertEqual(return_info.returncode, 2) - # return_info = monitor_snapshots_command() - # self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") - if __name__ == '__main__': unittest.main() From b992beba19c20a2ee47028883e7735f6fe1a2f73 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:34:47 +0000 Subject: [PATCH 26/50] Add test for monitoring immediately after running --- tests/0_monitoring_tests/test_monitoring.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 47263e3..80ce911 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -51,9 +51,9 @@ class TestsWithZpool(unittest.TestCase): def tearDown(self): """Clean up on either passed or failed tests""" subprocess.run(["zpool", "export", pool_name1]) - subprocess.run(["rm", "export", pool_disk_image1]) + subprocess.run(["rm", pool_disk_image1]) subprocess.run(["zpool", "export", pool_name2]) - subprocess.run(["rm", "export", pool_disk_image2]) + subprocess.run(["rm", pool_disk_image2]) def test_with_zpool_no_snapshots(self): """Test what happens if there is a zpool, but with no snapshots""" @@ -63,11 +63,13 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") self.assertEqual(return_info.returncode, 2) - # Run sanoid and test again + def test_immediately_after_running_sanoid(self): + """Test immediately after running sanoid --cron""" + run_sanoid_cron_command() return_info = monitor_snapshots_command() self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") - self.assertEqual(return_info.returncode, 2) + self.assertEqual(return_info.returncode, 0) if __name__ == '__main__': From 6710b2d36fe2a08e26e53ae77371cc8aa4c9e0d2 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Thu, 17 Mar 2022 22:38:41 +0000 Subject: [PATCH 27/50] Clear snapshot cache before each test --- tests/0_monitoring_tests/test_monitoring.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 80ce911..dd1490e 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -48,12 +48,15 @@ class TestsWithZpool(unittest.TestCase): subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) + # Clear the snapshot cache in between + subprocess.run(["rm", "-f", "/var/cache/sanoid/snapshots.txt"]) + def tearDown(self): """Clean up on either passed or failed tests""" subprocess.run(["zpool", "export", pool_name1]) - subprocess.run(["rm", pool_disk_image1]) + subprocess.run(["rm", "-f", pool_disk_image1]) subprocess.run(["zpool", "export", pool_name2]) - subprocess.run(["rm", pool_disk_image2]) + subprocess.run(["rm", "-f", pool_disk_image2]) def test_with_zpool_no_snapshots(self): """Test what happens if there is a zpool, but with no snapshots""" From 0b7c05f4c7d6a555f3d78a2d40f6c4d3556ad7a0 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 20:55:44 +0000 Subject: [PATCH 28/50] First time-based test (one warning) --- tests/0_monitoring_tests/run.sh | 6 ++--- tests/0_monitoring_tests/sanoid.conf | 23 +++++++++++++++- tests/0_monitoring_tests/test_monitoring.py | 29 ++++++++++++++++++++- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh index ec7fea1..68d872c 100755 --- a/tests/0_monitoring_tests/run.sh +++ b/tests/0_monitoring_tests/run.sh @@ -17,10 +17,10 @@ set -x # prepare setup checkEnvironment -# disableTimeSync +disableTimeSync -# # set timezone -# ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime # timestamp=$START diff --git a/tests/0_monitoring_tests/sanoid.conf b/tests/0_monitoring_tests/sanoid.conf index 9297d5d..fb84119 100644 --- a/tests/0_monitoring_tests/sanoid.conf +++ b/tests/0_monitoring_tests/sanoid.conf @@ -11,6 +11,27 @@ yearly = 0 autosnap = yes autoprune = no + hourly_warn = 90m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 32h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + [template_demo] - daily = 60 \ No newline at end of file + daily = 60 + hourly_warn = 290m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 48h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 \ No newline at end of file diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index dd1490e..bcbb45b 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -7,7 +7,7 @@ import os import subprocess -from tabnanny import check +import time import unittest @@ -28,6 +28,22 @@ def run_sanoid_cron_command(): return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) return return_info +def advance_time(seconds): + """Advances the system clock by seconds""" + + # Get the current time + clk_id = time.CLOCK_REALTIME + time_seconds = time.clock_gettime(clk_id) + print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + + # Set the clock to the current time plus seconds + time.clock_settime(clk_id, time_seconds + seconds) + + # Print the new time + time_seconds = time.clock_gettime(clk_id) + print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + return time_seconds + class TestMonitoringOutput(unittest.TestCase): def test_no_zpool(self): @@ -74,6 +90,17 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") self.assertEqual(return_info.returncode, 0) + def test_one_warning(self): + """Test one warning, no criticals""" + + run_sanoid_cron_command() + + # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else + advance_time(100 * 60) + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") + self.assertEqual(return_info.returncode, 0) + if __name__ == '__main__': unittest.main() From 2561a5ed5dfb42ab7c5f33470739bafd6a62186d Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:03:11 +0000 Subject: [PATCH 29/50] Add more info for debugging --- tests/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/README.md b/tests/README.md index 33be8f6..65e5658 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,11 +27,4 @@ cp sanoid/sanoid.defaults.conf /etc/sanoid/ ``` cd sanoid/tests/ ./run-tests.sh -``` - -### Example using LXD VMs ### -``` -VM_NAME=focal-sanoid-test -lxc init ubuntu:focal $VM_NAME --vm -c limits.cpu=8 -c limits.memory=10GB - ``` \ No newline at end of file From a2fc4b30ac4eca6fec15c04a7375d04cccd7b7ec Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:03:36 +0000 Subject: [PATCH 30/50] Add print for debugging --- tests/0_monitoring_tests/test_monitoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index bcbb45b..c18bbe7 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -98,6 +98,7 @@ class TestsWithZpool(unittest.TestCase): # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else advance_time(100 * 60) return_info = monitor_snapshots_command() + print(return_info.stdout) self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") self.assertEqual(return_info.returncode, 0) From f870d27e9f7016d46f5f4afde01438c0d1289485 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:16:31 +0000 Subject: [PATCH 31/50] Test warning string --- tests/0_monitoring_tests/test_monitoring.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index c18bbe7..b6a74e2 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -17,6 +17,9 @@ pool_name1 = "sanoid-test-1" pool_disk_image2 = "/zpool2.img" pool_name2 = "sanoid-test-2" +clk_id = time.CLOCK_REALTIME +starting_time = time.clock_gettime(clk_id) + def monitor_snapshots_command(): """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" @@ -73,6 +76,7 @@ class TestsWithZpool(unittest.TestCase): subprocess.run(["rm", "-f", pool_disk_image1]) subprocess.run(["zpool", "export", pool_name2]) subprocess.run(["rm", "-f", pool_disk_image2]) + time.clock_settime(clk_id, starting_time) def test_with_zpool_no_snapshots(self): """Test what happens if there is a zpool, but with no snapshots""" @@ -98,8 +102,12 @@ class TestsWithZpool(unittest.TestCase): # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else advance_time(100 * 60) return_info = monitor_snapshots_command() + # Output should be something like + # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # But we cannot be sure about the exact time print(return_info.stdout) - self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") + self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") + self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") self.assertEqual(return_info.returncode, 0) From d9da30dadd827a05700071843e5d44372ef1a918 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:20:18 +0000 Subject: [PATCH 32/50] Fix test for one monitoring warning. --- tests/0_monitoring_tests/test_monitoring.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index b6a74e2..09555a7 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -95,7 +95,7 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.returncode, 0) def test_one_warning(self): - """Test one warning, no criticals""" + """Test one warning, no criticals, to check output and error status""" run_sanoid_cron_command() @@ -105,10 +105,9 @@ class TestsWithZpool(unittest.TestCase): # Output should be something like # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n # But we cannot be sure about the exact time - print(return_info.stdout) self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") - self.assertEqual(return_info.returncode, 0) + self.assertEqual(return_info.returncode, 1) if __name__ == '__main__': From 4c9a9e2eb8d23eccf3c967de74cb6485277dc362 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:23:11 +0000 Subject: [PATCH 33/50] Use sanoid --force-update instead of manually deleting snapshot cache --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 09555a7..4162e0a 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -68,7 +68,7 @@ class TestsWithZpool(unittest.TestCase): subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) # Clear the snapshot cache in between - subprocess.run(["rm", "-f", "/var/cache/sanoid/snapshots.txt"]) + subprocess.run([sanoid_cmd, "--force-update"]) def tearDown(self): """Clean up on either passed or failed tests""" From adddd206723b0e6686356545972863e485355764 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:27:08 +0000 Subject: [PATCH 34/50] Add initial test for two critical monitoring errors --- tests/0_monitoring_tests/test_monitoring.py | 22 +++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 4162e0a..bd83c81 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -102,9 +102,27 @@ class TestsWithZpool(unittest.TestCase): # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else advance_time(100 * 60) return_info = monitor_snapshots_command() - # Output should be something like + # Output should be something like: # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n - # But we cannot be sure about the exact time + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") + self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.returncode, 1) + + def test_two_criticals(self): + """Test two criticals, to check output and error status""" + + run_sanoid_cron_command() + + # Advance 360 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 + advance_time(360 * 60) + return_info = monitor_snapshots_command() + print(return_info.stdout) + # Output should be something like: + # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") self.assertEqual(return_info.returncode, 1) From 7462a1a7fe74067b6392f2251b7ab75d9249c0cf Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:29:18 +0000 Subject: [PATCH 35/50] Adjust time slightly --- tests/0_monitoring_tests/test_monitoring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index bd83c81..abeda18 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -37,14 +37,14 @@ def advance_time(seconds): # Get the current time clk_id = time.CLOCK_REALTIME time_seconds = time.clock_gettime(clk_id) - print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") # Set the clock to the current time plus seconds time.clock_settime(clk_id, time_seconds + seconds) # Print the new time time_seconds = time.clock_gettime(clk_id) - print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") return time_seconds @@ -115,8 +115,8 @@ class TestsWithZpool(unittest.TestCase): run_sanoid_cron_command() - # Advance 360 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 - advance_time(360 * 60) + # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 + advance_time(390 * 60) return_info = monitor_snapshots_command() print(return_info.stdout) # Output should be something like: From a7fd743228482cc47dfb0c1c19d5e7918933873b Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:42:00 +0000 Subject: [PATCH 36/50] Add test details for two criticals --- tests/0_monitoring_tests/test_monitoring.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index abeda18..68b458b 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -107,7 +107,7 @@ class TestsWithZpool(unittest.TestCase): # But we cannot be sure about the exact time as test execution could be different on # different machines, so ignore the bits that may be different. self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") - self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.stdout[-31:], b"s old (should be < 1h 30m 0s)\n") self.assertEqual(return_info.returncode, 1) def test_two_criticals(self): @@ -120,11 +120,14 @@ class TestsWithZpool(unittest.TestCase): return_info = monitor_snapshots_command() print(return_info.stdout) # Output should be something like: - # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n # But we cannot be sure about the exact time as test execution could be different on # different machines, so ignore the bits that may be different. - self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") - self.assertEqual(return_info.stdout[54:], b"s old (should be < 1h 30m 0s)\n") + comma_location = return_info.stdout.find(b",") + self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)") self.assertEqual(return_info.returncode, 1) From f0fd435fa6823c24d148ab30feb050410cdb3254 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:44:20 +0000 Subject: [PATCH 37/50] Fix test_two_criticals output test --- tests/0_monitoring_tests/test_monitoring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 68b458b..2d8f219 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -107,7 +107,7 @@ class TestsWithZpool(unittest.TestCase): # But we cannot be sure about the exact time as test execution could be different on # different machines, so ignore the bits that may be different. self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") - self.assertEqual(return_info.stdout[-31:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") self.assertEqual(return_info.returncode, 1) def test_two_criticals(self): @@ -127,7 +127,7 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") - self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") self.assertEqual(return_info.returncode, 1) From 38ed37c96d45b90ee4014620cfe22840adcd71dd Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:45:44 +0000 Subject: [PATCH 38/50] Fix test_two_criticals returncode --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 2d8f219..99c9699 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -128,7 +128,7 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") - self.assertEqual(return_info.returncode, 1) + self.assertEqual(return_info.returncode, 2) if __name__ == '__main__': From ae4c0c6fa2e40453b76a02962bec894dda487d2f Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 21:53:39 +0000 Subject: [PATCH 39/50] Start test_two_warnings_daily --- tests/0_monitoring_tests/test_monitoring.py | 31 +++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 99c9699..4f7db63 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -94,8 +94,8 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") self.assertEqual(return_info.returncode, 0) - def test_one_warning(self): - """Test one warning, no criticals, to check output and error status""" + def test_one_warning_hourly(self): + """Test one warning on hourly snapshots, no critical warnings, to check output and error status""" run_sanoid_cron_command() @@ -110,15 +110,15 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") self.assertEqual(return_info.returncode, 1) - def test_two_criticals(self): - """Test two criticals, to check output and error status""" + def test_two_criticals_hourly(self): + """Test two criticals (hourly), to check output and error status""" run_sanoid_cron_command() # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 advance_time(390 * 60) return_info = monitor_snapshots_command() - print(return_info.stdout) + # Output should be something like: # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n # But we cannot be sure about the exact time as test execution could be different on @@ -130,6 +130,27 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") self.assertEqual(return_info.returncode, 2) + def test_two_warnings_daily(self): + """Test two warnings (daily), to check output and error status""" + + run_sanoid_cron_command() + + # Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2 + advance_time(29 * 60 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + print(return_info.stdout) + comma_location = return_info.stdout.find(b",") + self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") + self.assertEqual(return_info.returncode, 2) + if __name__ == '__main__': unittest.main() From c28f1c30dc4ea229dc1f9317d7b1929f6fcbc674 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 22:09:29 +0000 Subject: [PATCH 40/50] Fix test_two_warnings_daily text test --- tests/0_monitoring_tests/test_monitoring.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 4f7db63..4dc6615 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -140,16 +140,23 @@ class TestsWithZpool(unittest.TestCase): return_info = monitor_snapshots_command() # Output should be something like: - # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n + # CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n # But we cannot be sure about the exact time as test execution could be different on # different machines, so ignore the bits that may be different. print(return_info.stdout) - comma_location = return_info.stdout.find(b",") - self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") - self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") - self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") - self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") - self.assertEqual(return_info.returncode, 2) + output_list = return_info.stdout.split(b", ") + + self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ") + self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)") + self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ") + self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)") + + self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") + self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") + + self.assertEqual(return_info.returncode, 1) if __name__ == '__main__': From 81529523fd94496788fdbf867988755550bd8329 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 22:10:52 +0000 Subject: [PATCH 41/50] Fix return code --- tests/0_monitoring_tests/test_monitoring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 4dc6615..1b49e4d 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -153,10 +153,10 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") - self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-23 newest daily snapshot is 1d ") self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") - self.assertEqual(return_info.returncode, 1) + self.assertEqual(return_info.returncode, 2) if __name__ == '__main__': From c7e57b7c7472df6aafc88aec0e1955e81b0d6119 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Wed, 23 Mar 2022 22:11:45 +0000 Subject: [PATCH 42/50] Fix test_two_warnings --- tests/0_monitoring_tests/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 1b49e4d..5e30597 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -153,7 +153,7 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") - self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-23 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") self.assertEqual(return_info.returncode, 2) From 5015aecd7e0af8dcaca0fa8eab269fe2adc8d133 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 20:35:24 +0100 Subject: [PATCH 43/50] change name of test_two_warnings_daily to be more accurate --- tests/0_monitoring_tests/test_monitoring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py index 5e30597..ae42c4a 100644 --- a/tests/0_monitoring_tests/test_monitoring.py +++ b/tests/0_monitoring_tests/test_monitoring.py @@ -130,8 +130,8 @@ class TestsWithZpool(unittest.TestCase): self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") self.assertEqual(return_info.returncode, 2) - def test_two_warnings_daily(self): - """Test two warnings (daily), to check output and error status""" + def test_two_criticals_hourly_two_warnings_daily(self): + """Test two criticals (hourly) and two warnings (daily), to check output and error status""" run_sanoid_cron_command() From cdbe07e4ec794707320ce43beb56a16c9e02c282 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 21:28:21 +0100 Subject: [PATCH 44/50] Create a new set of tests for --monitor-snapshots these create pools in a number of states and check the output text and return code --- tests/3_monitor_snapshots/run.sh | 18 ++ tests/3_monitor_snapshots/sanoid.conf | 37 +++++ tests/3_monitor_snapshots/test_monitoring.py | 163 +++++++++++++++++++ tests/README.md | 31 ++++ tests/common/lib.sh | 2 +- tests/run-tests.sh | 2 +- 6 files changed, 251 insertions(+), 2 deletions(-) create mode 100755 tests/3_monitor_snapshots/run.sh create mode 100644 tests/3_monitor_snapshots/sanoid.conf create mode 100644 tests/3_monitor_snapshots/test_monitoring.py create mode 100644 tests/README.md diff --git a/tests/3_monitor_snapshots/run.sh b/tests/3_monitor_snapshots/run.sh new file mode 100755 index 0000000..0ae08f1 --- /dev/null +++ b/tests/3_monitor_snapshots/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -x + +# this test will create pools in a number of states +# and check the output text and return code of +# sanoid --monitor-snapshots + +. ../common/lib.sh + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +python3 test_monitoring.py diff --git a/tests/3_monitor_snapshots/sanoid.conf b/tests/3_monitor_snapshots/sanoid.conf new file mode 100644 index 0000000..fb84119 --- /dev/null +++ b/tests/3_monitor_snapshots/sanoid.conf @@ -0,0 +1,37 @@ +[sanoid-test-1] + use_template = production + +[sanoid-test-2] + use_template = demo + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no + hourly_warn = 90m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 32h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + + +[template_demo] + daily = 60 + hourly_warn = 290m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 48h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 \ No newline at end of file diff --git a/tests/3_monitor_snapshots/test_monitoring.py b/tests/3_monitor_snapshots/test_monitoring.py new file mode 100644 index 0000000..ae42c4a --- /dev/null +++ b/tests/3_monitor_snapshots/test_monitoring.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved +# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this +# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. + + +import os +import subprocess +import time +import unittest + + +sanoid_cmd = os.environ.get("SANOID") +pool_disk_image1 = "/zpool1.img" +pool_name1 = "sanoid-test-1" +pool_disk_image2 = "/zpool2.img" +pool_name2 = "sanoid-test-2" + +clk_id = time.CLOCK_REALTIME +starting_time = time.clock_gettime(clk_id) + + +def monitor_snapshots_command(): + """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) + return return_info + +def run_sanoid_cron_command(): + """Runs sanoid and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) + return return_info + +def advance_time(seconds): + """Advances the system clock by seconds""" + + # Get the current time + clk_id = time.CLOCK_REALTIME + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + + # Set the clock to the current time plus seconds + time.clock_settime(clk_id, time_seconds + seconds) + + # Print the new time + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + return time_seconds + + +class TestMonitoringOutput(unittest.TestCase): + def test_no_zpool(self): + """Test what happens if there is no zpool at all""" + + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 2) + +class TestsWithZpool(unittest.TestCase): + """Tests that require a test zpool""" + + def setUp(self): + """Set up the zpool""" + subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True) + subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True) + + subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) + subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) + + # Clear the snapshot cache in between + subprocess.run([sanoid_cmd, "--force-update"]) + + def tearDown(self): + """Clean up on either passed or failed tests""" + subprocess.run(["zpool", "export", pool_name1]) + subprocess.run(["rm", "-f", pool_disk_image1]) + subprocess.run(["zpool", "export", pool_name2]) + subprocess.run(["rm", "-f", pool_disk_image2]) + time.clock_settime(clk_id, starting_time) + + def test_with_zpool_no_snapshots(self): + """Test what happens if there is a zpool, but with no snapshots""" + + # Run sanoid --monitor-snapshots before doing anything else + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 2) + + def test_immediately_after_running_sanoid(self): + """Test immediately after running sanoid --cron""" + + run_sanoid_cron_command() + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") + self.assertEqual(return_info.returncode, 0) + + def test_one_warning_hourly(self): + """Test one warning on hourly snapshots, no critical warnings, to check output and error status""" + + run_sanoid_cron_command() + + # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else + advance_time(100 * 60) + return_info = monitor_snapshots_command() + # Output should be something like: + # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") + self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.returncode, 1) + + def test_two_criticals_hourly(self): + """Test two criticals (hourly), to check output and error status""" + + run_sanoid_cron_command() + + # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 + advance_time(390 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + comma_location = return_info.stdout.find(b",") + self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") + self.assertEqual(return_info.returncode, 2) + + def test_two_criticals_hourly_two_warnings_daily(self): + """Test two criticals (hourly) and two warnings (daily), to check output and error status""" + + run_sanoid_cron_command() + + # Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2 + advance_time(29 * 60 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + print(return_info.stdout) + output_list = return_info.stdout.split(b", ") + + self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ") + self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)") + self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ") + self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)") + + self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") + self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") + + self.assertEqual(return_info.returncode, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..2e7ad56 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,31 @@ +### Requirements ### +Tests must be run inside a virtual machine. This is for your own safety, as the tests may create and destroy zpools etc. + +A VM with 35GB of storage and 8 cores completes the tests in about 5 hours. + +#### Packages #### +The tests require the following packages to be installed in the VM (Ubuntu 20.04 package names are used, translate as appropriate): +``` +zfsutils-linux +libconfig-inifiles-perl +libcapture-tiny-perl +``` +``` +apt install zfsutils-linux libconfig-inifiles-perl libcapture-tiny-perl +``` + +#### Install sanoid within the VM #### +Install sanoid within the VM, for example +``` +apt install git +git clone https://github.com/jimsalterjrs/sanoid.git +mkdir /etc/sanoid/ +cp sanoid/sanoid.defaults.conf /etc/sanoid/ +``` + +### Run the tests ## +This requires root/sudo privileges. +``` +cd sanoid/tests/ +./run-tests.sh +``` \ No newline at end of file diff --git a/tests/common/lib.sh b/tests/common/lib.sh index 904c98f..89d1238 100644 --- a/tests/common/lib.sh +++ b/tests/common/lib.sh @@ -10,7 +10,7 @@ function setup { export SANOID="../../sanoid" # make sure that there is no cache file - rm -f /var/cache/sanoidsnapshots.txt + rm -f /var/cache/sanoid/snapshots.txt # install needed sanoid configuration files [ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 38054b0..9a2a080 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -# run's all the available tests +# runs all the available tests for test in */; do if [ ! -x "${test}/run.sh" ]; then From 9cc6b7f12b0e9b3622448bed6c933bd8da4f86d9 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 21:36:19 +0100 Subject: [PATCH 45/50] Delete redundant set of tests from rename/merge --- tests/0_monitoring_tests/run.sh | 34 ---- tests/0_monitoring_tests/sanoid.conf | 37 ----- tests/0_monitoring_tests/test_monitoring.py | 163 -------------------- 3 files changed, 234 deletions(-) delete mode 100755 tests/0_monitoring_tests/run.sh delete mode 100644 tests/0_monitoring_tests/sanoid.conf delete mode 100644 tests/0_monitoring_tests/test_monitoring.py diff --git a/tests/0_monitoring_tests/run.sh b/tests/0_monitoring_tests/run.sh deleted file mode 100755 index 68d872c..0000000 --- a/tests/0_monitoring_tests/run.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -x - -# this test will take hourly, daily and monthly snapshots -# for the whole year of 2017 in the timezone Europe/Vienna -# sanoid is run hourly and no snapshots are pruned - -. ../common/lib.sh - -# POOL_NAME="sanoid-test-1" -# POOL_TARGET="" # root - -# # UTC timestamp of start and end -# START="1483225200" -# END="1514761199" - -# prepare -setup -checkEnvironment -disableTimeSync - -# set timezone -ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime - -# timestamp=$START - -# function cleanUp { -# zpool export "${POOL_NAME}" -# } - -# export pool in any case -# trap cleanUp EXIT - -python3 test_monitoring.py diff --git a/tests/0_monitoring_tests/sanoid.conf b/tests/0_monitoring_tests/sanoid.conf deleted file mode 100644 index fb84119..0000000 --- a/tests/0_monitoring_tests/sanoid.conf +++ /dev/null @@ -1,37 +0,0 @@ -[sanoid-test-1] - use_template = production - -[sanoid-test-2] - use_template = demo - -[template_production] - hourly = 36 - daily = 30 - monthly = 3 - yearly = 0 - autosnap = yes - autoprune = no - hourly_warn = 90m - hourly_crit = 360m - daily_warn = 28h - daily_crit = 32h - weekly_warn = 0 - weekly_crit = 0 - monthly_warn = 32d - monthly_crit = 40d - yearly_warn = 0 - yearly_crit = 0 - - -[template_demo] - daily = 60 - hourly_warn = 290m - hourly_crit = 360m - daily_warn = 28h - daily_crit = 48h - weekly_warn = 0 - weekly_crit = 0 - monthly_warn = 32d - monthly_crit = 40d - yearly_warn = 0 - yearly_crit = 0 \ No newline at end of file diff --git a/tests/0_monitoring_tests/test_monitoring.py b/tests/0_monitoring_tests/test_monitoring.py deleted file mode 100644 index ae42c4a..0000000 --- a/tests/0_monitoring_tests/test_monitoring.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 - -# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved -# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this -# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. - - -import os -import subprocess -import time -import unittest - - -sanoid_cmd = os.environ.get("SANOID") -pool_disk_image1 = "/zpool1.img" -pool_name1 = "sanoid-test-1" -pool_disk_image2 = "/zpool2.img" -pool_name2 = "sanoid-test-2" - -clk_id = time.CLOCK_REALTIME -starting_time = time.clock_gettime(clk_id) - - -def monitor_snapshots_command(): - """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" - return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) - return return_info - -def run_sanoid_cron_command(): - """Runs sanoid and returns a CompletedProcess instance""" - return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) - return return_info - -def advance_time(seconds): - """Advances the system clock by seconds""" - - # Get the current time - clk_id = time.CLOCK_REALTIME - time_seconds = time.clock_gettime(clk_id) - # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") - - # Set the clock to the current time plus seconds - time.clock_settime(clk_id, time_seconds + seconds) - - # Print the new time - time_seconds = time.clock_gettime(clk_id) - # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") - return time_seconds - - -class TestMonitoringOutput(unittest.TestCase): - def test_no_zpool(self): - """Test what happens if there is no zpool at all""" - - return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") - self.assertEqual(return_info.returncode, 2) - -class TestsWithZpool(unittest.TestCase): - """Tests that require a test zpool""" - - def setUp(self): - """Set up the zpool""" - subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True) - subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True) - - subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) - subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) - - # Clear the snapshot cache in between - subprocess.run([sanoid_cmd, "--force-update"]) - - def tearDown(self): - """Clean up on either passed or failed tests""" - subprocess.run(["zpool", "export", pool_name1]) - subprocess.run(["rm", "-f", pool_disk_image1]) - subprocess.run(["zpool", "export", pool_name2]) - subprocess.run(["rm", "-f", pool_disk_image2]) - time.clock_settime(clk_id, starting_time) - - def test_with_zpool_no_snapshots(self): - """Test what happens if there is a zpool, but with no snapshots""" - - # Run sanoid --monitor-snapshots before doing anything else - return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") - self.assertEqual(return_info.returncode, 2) - - def test_immediately_after_running_sanoid(self): - """Test immediately after running sanoid --cron""" - - run_sanoid_cron_command() - return_info = monitor_snapshots_command() - self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") - self.assertEqual(return_info.returncode, 0) - - def test_one_warning_hourly(self): - """Test one warning on hourly snapshots, no critical warnings, to check output and error status""" - - run_sanoid_cron_command() - - # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else - advance_time(100 * 60) - return_info = monitor_snapshots_command() - # Output should be something like: - # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n - # But we cannot be sure about the exact time as test execution could be different on - # different machines, so ignore the bits that may be different. - self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") - self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") - self.assertEqual(return_info.returncode, 1) - - def test_two_criticals_hourly(self): - """Test two criticals (hourly), to check output and error status""" - - run_sanoid_cron_command() - - # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 - advance_time(390 * 60) - return_info = monitor_snapshots_command() - - # Output should be something like: - # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n - # But we cannot be sure about the exact time as test execution could be different on - # different machines, so ignore the bits that may be different. - comma_location = return_info.stdout.find(b",") - self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") - self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") - self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") - self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") - self.assertEqual(return_info.returncode, 2) - - def test_two_criticals_hourly_two_warnings_daily(self): - """Test two criticals (hourly) and two warnings (daily), to check output and error status""" - - run_sanoid_cron_command() - - # Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2 - advance_time(29 * 60 * 60) - return_info = monitor_snapshots_command() - - # Output should be something like: - # CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n - # But we cannot be sure about the exact time as test execution could be different on - # different machines, so ignore the bits that may be different. - print(return_info.stdout) - output_list = return_info.stdout.split(b", ") - - self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ") - self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)") - self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ") - self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)") - - self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") - self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") - self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") - self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") - - self.assertEqual(return_info.returncode, 2) - - -if __name__ == '__main__': - unittest.main() From 9759fa45c49364e97e7d3e4c29c66dfa16c19ef0 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 22:10:03 +0100 Subject: [PATCH 46/50] Ready to merge --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 2e7ad56..3acfbbe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,7 +1,7 @@ ### Requirements ### Tests must be run inside a virtual machine. This is for your own safety, as the tests may create and destroy zpools etc. -A VM with 35GB of storage and 8 cores completes the tests in about 5 hours. +A VM with 35GB of storage and 8 cores running Ubuntu 20.04 completes the tests in about 5 hours. #### Packages #### The tests require the following packages to be installed in the VM (Ubuntu 20.04 package names are used, translate as appropriate): From b17a0fe10418c6b6c63c477fb08c0e3173d2a9ca Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 22:13:49 +0100 Subject: [PATCH 47/50] Add newline at end of sanoid.conf --- tests/3_monitor_snapshots/sanoid.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/3_monitor_snapshots/sanoid.conf b/tests/3_monitor_snapshots/sanoid.conf index fb84119..ef51ad3 100644 --- a/tests/3_monitor_snapshots/sanoid.conf +++ b/tests/3_monitor_snapshots/sanoid.conf @@ -34,4 +34,5 @@ monthly_warn = 32d monthly_crit = 40d yearly_warn = 0 - yearly_crit = 0 \ No newline at end of file + yearly_crit = 0 + \ No newline at end of file From fec7cf9e18207ee810dc2abc186d987805c5a05c Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sun, 27 Mar 2022 22:23:03 +0100 Subject: [PATCH 48/50] Add newline --- tests/3_monitor_snapshots/sanoid.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/3_monitor_snapshots/sanoid.conf b/tests/3_monitor_snapshots/sanoid.conf index ef51ad3..c398b69 100644 --- a/tests/3_monitor_snapshots/sanoid.conf +++ b/tests/3_monitor_snapshots/sanoid.conf @@ -35,4 +35,3 @@ monthly_crit = 40d yearly_warn = 0 yearly_crit = 0 - \ No newline at end of file From 135f310881c2820d150177299a4fbbd4214f7c95 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sat, 2 Apr 2022 21:23:34 +0100 Subject: [PATCH 49/50] Add copy of monitor snapshots test with monitor_dont_warn and monitor_dont_crit set --- .../4_monitor_snapshots_dont_warncrit/run.sh | 18 ++ .../sanoid.conf | 41 +++++ .../test_monitoring.py | 163 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100755 tests/4_monitor_snapshots_dont_warncrit/run.sh create mode 100644 tests/4_monitor_snapshots_dont_warncrit/sanoid.conf create mode 100644 tests/4_monitor_snapshots_dont_warncrit/test_monitoring.py diff --git a/tests/4_monitor_snapshots_dont_warncrit/run.sh b/tests/4_monitor_snapshots_dont_warncrit/run.sh new file mode 100755 index 0000000..0ae08f1 --- /dev/null +++ b/tests/4_monitor_snapshots_dont_warncrit/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -x + +# this test will create pools in a number of states +# and check the output text and return code of +# sanoid --monitor-snapshots + +. ../common/lib.sh + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +python3 test_monitoring.py diff --git a/tests/4_monitor_snapshots_dont_warncrit/sanoid.conf b/tests/4_monitor_snapshots_dont_warncrit/sanoid.conf new file mode 100644 index 0000000..5bcf810 --- /dev/null +++ b/tests/4_monitor_snapshots_dont_warncrit/sanoid.conf @@ -0,0 +1,41 @@ +[sanoid-test-1] + use_template = production + +[sanoid-test-2] + use_template = demo + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no + hourly_warn = 90m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 32h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + monitor_dont_warn = true + monitor_dont_crit = Yes + + +[template_demo] + daily = 60 + hourly_warn = 290m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 48h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + monitor_dont_warn = ON + monitor_dont_crit = 1 diff --git a/tests/4_monitor_snapshots_dont_warncrit/test_monitoring.py b/tests/4_monitor_snapshots_dont_warncrit/test_monitoring.py new file mode 100644 index 0000000..9a339b4 --- /dev/null +++ b/tests/4_monitor_snapshots_dont_warncrit/test_monitoring.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved +# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this +# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. + + +import os +import subprocess +import time +import unittest + + +sanoid_cmd = os.environ.get("SANOID") +pool_disk_image1 = "/zpool1.img" +pool_name1 = "sanoid-test-1" +pool_disk_image2 = "/zpool2.img" +pool_name2 = "sanoid-test-2" + +clk_id = time.CLOCK_REALTIME +starting_time = time.clock_gettime(clk_id) + + +def monitor_snapshots_command(): + """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) + return return_info + +def run_sanoid_cron_command(): + """Runs sanoid and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) + return return_info + +def advance_time(seconds): + """Advances the system clock by seconds""" + + # Get the current time + clk_id = time.CLOCK_REALTIME + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + + # Set the clock to the current time plus seconds + time.clock_settime(clk_id, time_seconds + seconds) + + # Print the new time + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + return time_seconds + + +class TestMonitoringOutput(unittest.TestCase): + def test_no_zpool(self): + """Test what happens if there is no zpool at all""" + + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 0) + +class TestsWithZpool(unittest.TestCase): + """Tests that require a test zpool""" + + def setUp(self): + """Set up the zpool""" + subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True) + subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True) + + subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) + subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) + + # Clear the snapshot cache in between + subprocess.run([sanoid_cmd, "--force-update"]) + + def tearDown(self): + """Clean up on either passed or failed tests""" + subprocess.run(["zpool", "export", pool_name1]) + subprocess.run(["rm", "-f", pool_disk_image1]) + subprocess.run(["zpool", "export", pool_name2]) + subprocess.run(["rm", "-f", pool_disk_image2]) + time.clock_settime(clk_id, starting_time) + + def test_with_zpool_no_snapshots(self): + """Test what happens if there is a zpool, but with no snapshots""" + + # Run sanoid --monitor-snapshots before doing anything else + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 0) + + def test_immediately_after_running_sanoid(self): + """Test immediately after running sanoid --cron""" + + run_sanoid_cron_command() + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") + self.assertEqual(return_info.returncode, 0) + + def test_one_warning_hourly(self): + """Test one warning on hourly snapshots, no critical warnings, to check output and error status""" + + run_sanoid_cron_command() + + # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else + advance_time(100 * 60) + return_info = monitor_snapshots_command() + # Output should be something like: + # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") + self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.returncode, 0) + + def test_two_criticals_hourly(self): + """Test two criticals (hourly), to check output and error status""" + + run_sanoid_cron_command() + + # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 + advance_time(390 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + comma_location = return_info.stdout.find(b",") + self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") + self.assertEqual(return_info.returncode, 0) + + def test_two_criticals_hourly_two_warnings_daily(self): + """Test two criticals (hourly) and two warnings (daily), to check output and error status""" + + run_sanoid_cron_command() + + # Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2 + advance_time(29 * 60 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + print(return_info.stdout) + output_list = return_info.stdout.split(b", ") + + self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ") + self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)") + self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ") + self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)") + + self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") + self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") + + self.assertEqual(return_info.returncode, 0) + + +if __name__ == '__main__': + unittest.main() From cd0b9f362bafd8d8e7ca384849ff161ec602dea9 Mon Sep 17 00:00:00 2001 From: Aaron Whitehouse Date: Sat, 2 Apr 2022 21:37:54 +0100 Subject: [PATCH 50/50] Add tests with monitor_dont_crit set but without monitor_dont_warn --- .../5_monitor_snapshots_warn_dont_crit/run.sh | 18 ++ .../sanoid.conf | 41 +++++ .../test_monitoring.py | 163 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100755 tests/5_monitor_snapshots_warn_dont_crit/run.sh create mode 100644 tests/5_monitor_snapshots_warn_dont_crit/sanoid.conf create mode 100644 tests/5_monitor_snapshots_warn_dont_crit/test_monitoring.py diff --git a/tests/5_monitor_snapshots_warn_dont_crit/run.sh b/tests/5_monitor_snapshots_warn_dont_crit/run.sh new file mode 100755 index 0000000..0ae08f1 --- /dev/null +++ b/tests/5_monitor_snapshots_warn_dont_crit/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -x + +# this test will create pools in a number of states +# and check the output text and return code of +# sanoid --monitor-snapshots + +. ../common/lib.sh + +# prepare +setup +checkEnvironment +disableTimeSync + +# set timezone +ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime + +python3 test_monitoring.py diff --git a/tests/5_monitor_snapshots_warn_dont_crit/sanoid.conf b/tests/5_monitor_snapshots_warn_dont_crit/sanoid.conf new file mode 100644 index 0000000..1fed450 --- /dev/null +++ b/tests/5_monitor_snapshots_warn_dont_crit/sanoid.conf @@ -0,0 +1,41 @@ +[sanoid-test-1] + use_template = production + +[sanoid-test-2] + use_template = demo + +[template_production] + hourly = 36 + daily = 30 + monthly = 3 + yearly = 0 + autosnap = yes + autoprune = no + hourly_warn = 90m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 32h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + monitor_dont_warn = FALSE + monitor_dont_crit = yes + + +[template_demo] + daily = 60 + hourly_warn = 290m + hourly_crit = 360m + daily_warn = 28h + daily_crit = 48h + weekly_warn = 0 + weekly_crit = 0 + monthly_warn = 32d + monthly_crit = 40d + yearly_warn = 0 + yearly_crit = 0 + monitor_dont_warn = Off + monitor_dont_crit = On diff --git a/tests/5_monitor_snapshots_warn_dont_crit/test_monitoring.py b/tests/5_monitor_snapshots_warn_dont_crit/test_monitoring.py new file mode 100644 index 0000000..4c0a907 --- /dev/null +++ b/tests/5_monitor_snapshots_warn_dont_crit/test_monitoring.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved +# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this +# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. + + +import os +import subprocess +import time +import unittest + + +sanoid_cmd = os.environ.get("SANOID") +pool_disk_image1 = "/zpool1.img" +pool_name1 = "sanoid-test-1" +pool_disk_image2 = "/zpool2.img" +pool_name2 = "sanoid-test-2" + +clk_id = time.CLOCK_REALTIME +starting_time = time.clock_gettime(clk_id) + + +def monitor_snapshots_command(): + """Runs sanoid --monitor-snapshots and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True) + return return_info + +def run_sanoid_cron_command(): + """Runs sanoid and returns a CompletedProcess instance""" + return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True) + return return_info + +def advance_time(seconds): + """Advances the system clock by seconds""" + + # Get the current time + clk_id = time.CLOCK_REALTIME + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + + # Set the clock to the current time plus seconds + time.clock_settime(clk_id, time_seconds + seconds) + + # Print the new time + time_seconds = time.clock_gettime(clk_id) + # print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT") + return time_seconds + + +class TestMonitoringOutput(unittest.TestCase): + def test_no_zpool(self): + """Test what happens if there is no zpool at all""" + + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 0) + +class TestsWithZpool(unittest.TestCase): + """Tests that require a test zpool""" + + def setUp(self): + """Set up the zpool""" + subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True) + subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True) + + subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True) + subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True) + + # Clear the snapshot cache in between + subprocess.run([sanoid_cmd, "--force-update"]) + + def tearDown(self): + """Clean up on either passed or failed tests""" + subprocess.run(["zpool", "export", pool_name1]) + subprocess.run(["rm", "-f", pool_disk_image1]) + subprocess.run(["zpool", "export", pool_name2]) + subprocess.run(["rm", "-f", pool_disk_image2]) + time.clock_settime(clk_id, starting_time) + + def test_with_zpool_no_snapshots(self): + """Test what happens if there is a zpool, but with no snapshots""" + + # Run sanoid --monitor-snapshots before doing anything else + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n") + self.assertEqual(return_info.returncode, 0) + + def test_immediately_after_running_sanoid(self): + """Test immediately after running sanoid --cron""" + + run_sanoid_cron_command() + return_info = monitor_snapshots_command() + self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n") + self.assertEqual(return_info.returncode, 0) + + def test_one_warning_hourly(self): + """Test one warning on hourly snapshots, no critical warnings, to check output and error status""" + + run_sanoid_cron_command() + + # Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else + advance_time(100 * 60) + return_info = monitor_snapshots_command() + # Output should be something like: + # WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ") + self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n") + self.assertEqual(return_info.returncode, 1) + + def test_two_criticals_hourly(self): + """Test two criticals (hourly), to check output and error status""" + + run_sanoid_cron_command() + + # Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2 + advance_time(390 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + comma_location = return_info.stdout.find(b",") + self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)") + self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ") + self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n") + self.assertEqual(return_info.returncode, 0) + + def test_two_criticals_hourly_two_warnings_daily(self): + """Test two criticals (hourly) and two warnings (daily), to check output and error status""" + + run_sanoid_cron_command() + + # Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2 + advance_time(29 * 60 * 60) + return_info = monitor_snapshots_command() + + # Output should be something like: + # CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n + # But we cannot be sure about the exact time as test execution could be different on + # different machines, so ignore the bits that may be different. + print(return_info.stdout) + output_list = return_info.stdout.split(b", ") + + self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ") + self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)") + self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ") + self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)") + + self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ") + self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)") + self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ") + self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n") + + self.assertEqual(return_info.returncode, 1) + + +if __name__ == '__main__': + unittest.main()