Compare commits
52 Commits
accd8ccbac
...
4c1de8017a
Author | SHA1 | Date |
---|---|---|
![]() |
4c1de8017a | |
![]() |
cd0b9f362b | |
![]() |
135f310881 | |
![]() |
fec7cf9e18 | |
![]() |
b17a0fe104 | |
![]() |
9759fa45c4 | |
![]() |
9cc6b7f12b | |
![]() |
3a4e043c96 | |
![]() |
cdbe07e4ec | |
![]() |
5015aecd7e | |
![]() |
c7e57b7c74 | |
![]() |
81529523fd | |
![]() |
c28f1c30dc | |
![]() |
ae4c0c6fa2 | |
![]() |
38ed37c96d | |
![]() |
f0fd435fa6 | |
![]() |
a7fd743228 | |
![]() |
7462a1a7fe | |
![]() |
adddd20672 | |
![]() |
4c9a9e2eb8 | |
![]() |
d9da30dadd | |
![]() |
f870d27e9f | |
![]() |
a2fc4b30ac | |
![]() |
2561a5ed5d | |
![]() |
0b7c05f4c7 | |
![]() |
6710b2d36f | |
![]() |
b992beba19 | |
![]() |
d0f89c96bb | |
![]() |
02aedfaaa2 | |
![]() |
867b755bb1 | |
![]() |
49589e2a17 | |
![]() |
33648050dd | |
![]() |
8571510685 | |
![]() |
3e9730d135 | |
![]() |
890c785d0b | |
![]() |
37bce97d6e | |
![]() |
e556495dea | |
![]() |
c8f2b3f283 | |
![]() |
501c9921c4 | |
![]() |
26777c2c68 | |
![]() |
f93985111f | |
![]() |
c488df5983 | |
![]() |
8bb8d38cc8 | |
![]() |
c21d8d0168 | |
![]() |
123e35e804 | |
![]() |
1bdb60715c | |
![]() |
3655c02775 | |
![]() |
b3ee98a234 | |
![]() |
372eadcb37 | |
![]() |
e26671c97d | |
![]() |
cdbc1915ba | |
![]() |
e963dd228f |
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# this test will create pools in a number of states
|
||||||
|
# and check the output text and return code of
|
||||||
|
# sanoid --monitor-snapshots
|
||||||
|
|
||||||
|
. ../common/lib.sh
|
||||||
|
|
||||||
|
# prepare
|
||||||
|
setup
|
||||||
|
checkEnvironment
|
||||||
|
disableTimeSync
|
||||||
|
|
||||||
|
# set timezone
|
||||||
|
ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
|
||||||
|
|
||||||
|
python3 test_monitoring.py
|
|
@ -0,0 +1,37 @@
|
||||||
|
[sanoid-test-1]
|
||||||
|
use_template = production
|
||||||
|
|
||||||
|
[sanoid-test-2]
|
||||||
|
use_template = demo
|
||||||
|
|
||||||
|
[template_production]
|
||||||
|
hourly = 36
|
||||||
|
daily = 30
|
||||||
|
monthly = 3
|
||||||
|
yearly = 0
|
||||||
|
autosnap = yes
|
||||||
|
autoprune = no
|
||||||
|
hourly_warn = 90m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 32h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
||||||
|
|
||||||
|
|
||||||
|
[template_demo]
|
||||||
|
daily = 60
|
||||||
|
hourly_warn = 290m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 48h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
|
@ -0,0 +1,163 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
|
||||||
|
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
|
||||||
|
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
sanoid_cmd = os.environ.get("SANOID")
|
||||||
|
pool_disk_image1 = "/zpool1.img"
|
||||||
|
pool_name1 = "sanoid-test-1"
|
||||||
|
pool_disk_image2 = "/zpool2.img"
|
||||||
|
pool_name2 = "sanoid-test-2"
|
||||||
|
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
starting_time = time.clock_gettime(clk_id)
|
||||||
|
|
||||||
|
|
||||||
|
def monitor_snapshots_command():
|
||||||
|
"""Runs sanoid --monitor-snapshots and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def run_sanoid_cron_command():
|
||||||
|
"""Runs sanoid and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def advance_time(seconds):
|
||||||
|
"""Advances the system clock by seconds"""
|
||||||
|
|
||||||
|
# Get the current time
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
|
||||||
|
# Set the clock to the current time plus seconds
|
||||||
|
time.clock_settime(clk_id, time_seconds + seconds)
|
||||||
|
|
||||||
|
# Print the new time
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
return time_seconds
|
||||||
|
|
||||||
|
|
||||||
|
class TestMonitoringOutput(unittest.TestCase):
|
||||||
|
def test_no_zpool(self):
|
||||||
|
"""Test what happens if there is no zpool at all"""
|
||||||
|
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 2)
|
||||||
|
|
||||||
|
class TestsWithZpool(unittest.TestCase):
|
||||||
|
"""Tests that require a test zpool"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
"""Set up the zpool"""
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True)
|
||||||
|
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True)
|
||||||
|
|
||||||
|
# Clear the snapshot cache in between
|
||||||
|
subprocess.run([sanoid_cmd, "--force-update"])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
"""Clean up on either passed or failed tests"""
|
||||||
|
subprocess.run(["zpool", "export", pool_name1])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image1])
|
||||||
|
subprocess.run(["zpool", "export", pool_name2])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image2])
|
||||||
|
time.clock_settime(clk_id, starting_time)
|
||||||
|
|
||||||
|
def test_with_zpool_no_snapshots(self):
|
||||||
|
"""Test what happens if there is a zpool, but with no snapshots"""
|
||||||
|
|
||||||
|
# Run sanoid --monitor-snapshots before doing anything else
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 2)
|
||||||
|
|
||||||
|
def test_immediately_after_running_sanoid(self):
|
||||||
|
"""Test immediately after running sanoid --cron"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_one_warning_hourly(self):
|
||||||
|
"""Test one warning on hourly snapshots, no critical warnings, to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else
|
||||||
|
advance_time(100 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
# Output should be something like:
|
||||||
|
# WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ")
|
||||||
|
self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 1)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly(self):
|
||||||
|
"""Test two criticals (hourly), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(390 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
comma_location = return_info.stdout.find(b",")
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 2)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly_two_warnings_daily(self):
|
||||||
|
"""Test two criticals (hourly) and two warnings (daily), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(29 * 60 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
print(return_info.stdout)
|
||||||
|
output_list = return_info.stdout.split(b", ")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n")
|
||||||
|
|
||||||
|
self.assertEqual(return_info.returncode, 2)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# this test will create pools in a number of states
|
||||||
|
# and check the output text and return code of
|
||||||
|
# sanoid --monitor-snapshots
|
||||||
|
|
||||||
|
. ../common/lib.sh
|
||||||
|
|
||||||
|
# prepare
|
||||||
|
setup
|
||||||
|
checkEnvironment
|
||||||
|
disableTimeSync
|
||||||
|
|
||||||
|
# set timezone
|
||||||
|
ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
|
||||||
|
|
||||||
|
python3 test_monitoring.py
|
|
@ -0,0 +1,41 @@
|
||||||
|
[sanoid-test-1]
|
||||||
|
use_template = production
|
||||||
|
|
||||||
|
[sanoid-test-2]
|
||||||
|
use_template = demo
|
||||||
|
|
||||||
|
[template_production]
|
||||||
|
hourly = 36
|
||||||
|
daily = 30
|
||||||
|
monthly = 3
|
||||||
|
yearly = 0
|
||||||
|
autosnap = yes
|
||||||
|
autoprune = no
|
||||||
|
hourly_warn = 90m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 32h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
||||||
|
monitor_dont_warn = true
|
||||||
|
monitor_dont_crit = Yes
|
||||||
|
|
||||||
|
|
||||||
|
[template_demo]
|
||||||
|
daily = 60
|
||||||
|
hourly_warn = 290m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 48h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
||||||
|
monitor_dont_warn = ON
|
||||||
|
monitor_dont_crit = 1
|
|
@ -0,0 +1,163 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
|
||||||
|
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
|
||||||
|
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
sanoid_cmd = os.environ.get("SANOID")
|
||||||
|
pool_disk_image1 = "/zpool1.img"
|
||||||
|
pool_name1 = "sanoid-test-1"
|
||||||
|
pool_disk_image2 = "/zpool2.img"
|
||||||
|
pool_name2 = "sanoid-test-2"
|
||||||
|
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
starting_time = time.clock_gettime(clk_id)
|
||||||
|
|
||||||
|
|
||||||
|
def monitor_snapshots_command():
|
||||||
|
"""Runs sanoid --monitor-snapshots and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def run_sanoid_cron_command():
|
||||||
|
"""Runs sanoid and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def advance_time(seconds):
|
||||||
|
"""Advances the system clock by seconds"""
|
||||||
|
|
||||||
|
# Get the current time
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
|
||||||
|
# Set the clock to the current time plus seconds
|
||||||
|
time.clock_settime(clk_id, time_seconds + seconds)
|
||||||
|
|
||||||
|
# Print the new time
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
return time_seconds
|
||||||
|
|
||||||
|
|
||||||
|
class TestMonitoringOutput(unittest.TestCase):
|
||||||
|
def test_no_zpool(self):
|
||||||
|
"""Test what happens if there is no zpool at all"""
|
||||||
|
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
class TestsWithZpool(unittest.TestCase):
|
||||||
|
"""Tests that require a test zpool"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
"""Set up the zpool"""
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True)
|
||||||
|
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True)
|
||||||
|
|
||||||
|
# Clear the snapshot cache in between
|
||||||
|
subprocess.run([sanoid_cmd, "--force-update"])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
"""Clean up on either passed or failed tests"""
|
||||||
|
subprocess.run(["zpool", "export", pool_name1])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image1])
|
||||||
|
subprocess.run(["zpool", "export", pool_name2])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image2])
|
||||||
|
time.clock_settime(clk_id, starting_time)
|
||||||
|
|
||||||
|
def test_with_zpool_no_snapshots(self):
|
||||||
|
"""Test what happens if there is a zpool, but with no snapshots"""
|
||||||
|
|
||||||
|
# Run sanoid --monitor-snapshots before doing anything else
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_immediately_after_running_sanoid(self):
|
||||||
|
"""Test immediately after running sanoid --cron"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_one_warning_hourly(self):
|
||||||
|
"""Test one warning on hourly snapshots, no critical warnings, to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else
|
||||||
|
advance_time(100 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
# Output should be something like:
|
||||||
|
# WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ")
|
||||||
|
self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly(self):
|
||||||
|
"""Test two criticals (hourly), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(390 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
comma_location = return_info.stdout.find(b",")
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly_two_warnings_daily(self):
|
||||||
|
"""Test two criticals (hourly) and two warnings (daily), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(29 * 60 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
print(return_info.stdout)
|
||||||
|
output_list = return_info.stdout.split(b", ")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n")
|
||||||
|
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# this test will create pools in a number of states
|
||||||
|
# and check the output text and return code of
|
||||||
|
# sanoid --monitor-snapshots
|
||||||
|
|
||||||
|
. ../common/lib.sh
|
||||||
|
|
||||||
|
# prepare
|
||||||
|
setup
|
||||||
|
checkEnvironment
|
||||||
|
disableTimeSync
|
||||||
|
|
||||||
|
# set timezone
|
||||||
|
ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
|
||||||
|
|
||||||
|
python3 test_monitoring.py
|
|
@ -0,0 +1,41 @@
|
||||||
|
[sanoid-test-1]
|
||||||
|
use_template = production
|
||||||
|
|
||||||
|
[sanoid-test-2]
|
||||||
|
use_template = demo
|
||||||
|
|
||||||
|
[template_production]
|
||||||
|
hourly = 36
|
||||||
|
daily = 30
|
||||||
|
monthly = 3
|
||||||
|
yearly = 0
|
||||||
|
autosnap = yes
|
||||||
|
autoprune = no
|
||||||
|
hourly_warn = 90m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 32h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
||||||
|
monitor_dont_warn = FALSE
|
||||||
|
monitor_dont_crit = yes
|
||||||
|
|
||||||
|
|
||||||
|
[template_demo]
|
||||||
|
daily = 60
|
||||||
|
hourly_warn = 290m
|
||||||
|
hourly_crit = 360m
|
||||||
|
daily_warn = 28h
|
||||||
|
daily_crit = 48h
|
||||||
|
weekly_warn = 0
|
||||||
|
weekly_crit = 0
|
||||||
|
monthly_warn = 32d
|
||||||
|
monthly_crit = 40d
|
||||||
|
yearly_warn = 0
|
||||||
|
yearly_crit = 0
|
||||||
|
monitor_dont_warn = Off
|
||||||
|
monitor_dont_crit = On
|
|
@ -0,0 +1,163 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
|
||||||
|
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
|
||||||
|
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
sanoid_cmd = os.environ.get("SANOID")
|
||||||
|
pool_disk_image1 = "/zpool1.img"
|
||||||
|
pool_name1 = "sanoid-test-1"
|
||||||
|
pool_disk_image2 = "/zpool2.img"
|
||||||
|
pool_name2 = "sanoid-test-2"
|
||||||
|
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
starting_time = time.clock_gettime(clk_id)
|
||||||
|
|
||||||
|
|
||||||
|
def monitor_snapshots_command():
|
||||||
|
"""Runs sanoid --monitor-snapshots and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--monitor-snapshots"], capture_output=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def run_sanoid_cron_command():
|
||||||
|
"""Runs sanoid and returns a CompletedProcess instance"""
|
||||||
|
return_info = subprocess.run([sanoid_cmd, "--cron", "--verbose"], capture_output=True, check=True)
|
||||||
|
return return_info
|
||||||
|
|
||||||
|
def advance_time(seconds):
|
||||||
|
"""Advances the system clock by seconds"""
|
||||||
|
|
||||||
|
# Get the current time
|
||||||
|
clk_id = time.CLOCK_REALTIME
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
|
||||||
|
# Set the clock to the current time plus seconds
|
||||||
|
time.clock_settime(clk_id, time_seconds + seconds)
|
||||||
|
|
||||||
|
# Print the new time
|
||||||
|
time_seconds = time.clock_gettime(clk_id)
|
||||||
|
# print("Current unix time is", time_seconds, "or", time.asctime(time.gmtime(time_seconds)), "in GMT")
|
||||||
|
return time_seconds
|
||||||
|
|
||||||
|
|
||||||
|
class TestMonitoringOutput(unittest.TestCase):
|
||||||
|
def test_no_zpool(self):
|
||||||
|
"""Test what happens if there is no zpool at all"""
|
||||||
|
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
class TestsWithZpool(unittest.TestCase):
|
||||||
|
"""Tests that require a test zpool"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
"""Set up the zpool"""
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image1], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name1, pool_disk_image1], check=True)
|
||||||
|
|
||||||
|
subprocess.run(["truncate", "-s", "512M", pool_disk_image2], check=True)
|
||||||
|
subprocess.run(["zpool", "create", "-f", pool_name2, pool_disk_image2], check=True)
|
||||||
|
|
||||||
|
# Clear the snapshot cache in between
|
||||||
|
subprocess.run([sanoid_cmd, "--force-update"])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
"""Clean up on either passed or failed tests"""
|
||||||
|
subprocess.run(["zpool", "export", pool_name1])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image1])
|
||||||
|
subprocess.run(["zpool", "export", pool_name2])
|
||||||
|
subprocess.run(["rm", "-f", pool_disk_image2])
|
||||||
|
time.clock_settime(clk_id, starting_time)
|
||||||
|
|
||||||
|
def test_with_zpool_no_snapshots(self):
|
||||||
|
"""Test what happens if there is a zpool, but with no snapshots"""
|
||||||
|
|
||||||
|
# Run sanoid --monitor-snapshots before doing anything else
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"CRIT: sanoid-test-1 has no daily snapshots at all!, CRIT: sanoid-test-1 has no hourly snapshots at all!, CRIT: sanoid-test-1 has no monthly snapshots at all!, CRIT: sanoid-test-2 has no daily snapshots at all!, CRIT: sanoid-test-2 has no hourly snapshots at all!, CRIT: sanoid-test-2 has no monthly snapshots at all!\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_immediately_after_running_sanoid(self):
|
||||||
|
"""Test immediately after running sanoid --cron"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
self.assertEqual(return_info.stdout, b"OK: all monitored datasets (sanoid-test-1, sanoid-test-2) have fresh snapshots\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_one_warning_hourly(self):
|
||||||
|
"""Test one warning on hourly snapshots, no critical warnings, to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 100 mins to trigger the hourly warning on sanoid-test-1 but nothing else
|
||||||
|
advance_time(100 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
# Output should be something like:
|
||||||
|
# WARN: sanoid-test-1 newest hourly snapshot is 1h 40m 0s old (should be < 1h 30m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"WARN: sanoid-test-1 newest hourly snapshot is 1h ")
|
||||||
|
self.assertEqual(return_info.stdout[-30:], b"s old (should be < 1h 30m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 1)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly(self):
|
||||||
|
"""Test two criticals (hourly), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance 390 mins to trigger the hourly critical on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(390 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 6h 30m 1s old (should be < 6h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
comma_location = return_info.stdout.find(b",")
|
||||||
|
self.assertEqual(return_info.stdout[:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location - 28:comma_location], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(return_info.stdout[comma_location:comma_location + 51], b", CRIT: sanoid-test-2 newest hourly snapshot is 6h ")
|
||||||
|
self.assertEqual(return_info.stdout[-29:], b"s old (should be < 6h 0m 0s)\n")
|
||||||
|
self.assertEqual(return_info.returncode, 0)
|
||||||
|
|
||||||
|
def test_two_criticals_hourly_two_warnings_daily(self):
|
||||||
|
"""Test two criticals (hourly) and two warnings (daily), to check output and error status"""
|
||||||
|
|
||||||
|
run_sanoid_cron_command()
|
||||||
|
|
||||||
|
# Advance more than 28 hours to trigger the daily warning on both sanoid-test-1 and sanoid-test-2
|
||||||
|
advance_time(29 * 60 * 60)
|
||||||
|
return_info = monitor_snapshots_command()
|
||||||
|
|
||||||
|
# Output should be something like:
|
||||||
|
# CRIT: sanoid-test-1 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), CRIT: sanoid-test-2 newest hourly snapshot is 1d 5h 0m 0s old (should be < 6h 0m 0s), WARN: sanoid-test-1 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s), WARN: sanoid-test-2 newest daily snapshot is 1d 5h 0m 0s old (should be < 1d 4h 0m 0s)\n
|
||||||
|
# But we cannot be sure about the exact time as test execution could be different on
|
||||||
|
# different machines, so ignore the bits that may be different.
|
||||||
|
print(return_info.stdout)
|
||||||
|
output_list = return_info.stdout.split(b", ")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[0][:49], b"CRIT: sanoid-test-1 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[0][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[1][:49], b"CRIT: sanoid-test-2 newest hourly snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[1][-28:], b"s old (should be < 6h 0m 0s)")
|
||||||
|
|
||||||
|
self.assertEqual(output_list[2][:48], b"WARN: sanoid-test-1 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[2][-31:], b"s old (should be < 1d 4h 0m 0s)")
|
||||||
|
self.assertEqual(output_list[3][:48], b"WARN: sanoid-test-2 newest daily snapshot is 1d ")
|
||||||
|
self.assertEqual(output_list[3][-32:], b"s old (should be < 1d 4h 0m 0s)\n")
|
||||||
|
|
||||||
|
self.assertEqual(return_info.returncode, 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -0,0 +1,31 @@
|
||||||
|
### Requirements ###
|
||||||
|
Tests must be run inside a virtual machine. This is for your own safety, as the tests may create and destroy zpools etc.
|
||||||
|
|
||||||
|
A VM with 35GB of storage and 8 cores running Ubuntu 20.04 completes the tests in about 5 hours.
|
||||||
|
|
||||||
|
#### Packages ####
|
||||||
|
The tests require the following packages to be installed in the VM (Ubuntu 20.04 package names are used, translate as appropriate):
|
||||||
|
```
|
||||||
|
zfsutils-linux
|
||||||
|
libconfig-inifiles-perl
|
||||||
|
libcapture-tiny-perl
|
||||||
|
```
|
||||||
|
```
|
||||||
|
apt install zfsutils-linux libconfig-inifiles-perl libcapture-tiny-perl
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Install sanoid within the VM ####
|
||||||
|
Install sanoid within the VM, for example
|
||||||
|
```
|
||||||
|
apt install git
|
||||||
|
git clone https://github.com/jimsalterjrs/sanoid.git
|
||||||
|
mkdir /etc/sanoid/
|
||||||
|
cp sanoid/sanoid.defaults.conf /etc/sanoid/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run the tests ##
|
||||||
|
This requires root/sudo privileges.
|
||||||
|
```
|
||||||
|
cd sanoid/tests/
|
||||||
|
./run-tests.sh
|
||||||
|
```
|
|
@ -10,7 +10,7 @@ function setup {
|
||||||
export SANOID="../../sanoid"
|
export SANOID="../../sanoid"
|
||||||
|
|
||||||
# make sure that there is no cache file
|
# make sure that there is no cache file
|
||||||
rm -f /var/cache/sanoidsnapshots.txt
|
rm -f /var/cache/sanoid/snapshots.txt
|
||||||
|
|
||||||
# install needed sanoid configuration files
|
# install needed sanoid configuration files
|
||||||
[ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf
|
[ -f sanoid.conf ] && cp sanoid.conf /etc/sanoid/sanoid.conf
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# run's all the available tests
|
# runs all the available tests
|
||||||
|
|
||||||
for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g); do
|
for test in $(find . -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -g); do
|
||||||
if [ ! -x "${test}/run.sh" ]; then
|
if [ ! -x "${test}/run.sh" ]; then
|
||||||
|
|
Loading…
Reference in New Issue