Added Comet Backup plugin
This commit is contained in:
jackson
Kenyon Ralph
parent
eb9d7fa2a1
commit
836ec47f1f
|
|
@ -0,0 +1,9 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2022 Comet Licensing Ltd.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Munin Plugin for Comet Server
|
||||
|
||||
[](https://twitter.com/CometBackup)
|
||||
|
||||
This is a set of scripts for [Munin](https://munin-monitoring.org/) to export metrics from a running Comet Server instance over the Comet Server API.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.8.10 or later
|
||||
- Munin 2.0.69 or later
|
||||
|
||||
[Recommended Munin Ubuntu install instructions](https://www.hackerxone.com/2021/10/14/steps-to-install-munin-monitoring-tool-on-ubuntu-20-04-lts/)
|
||||
|
||||
## Installation instructions
|
||||
|
||||
The following instructions will cover installing the comet-munin plugin on Ubuntu 20.04.
|
||||
|
||||
A lot of the instructions here need to be run as root, so run `sudo -i`
|
||||
|
||||
1. Download, extract, and move the scripts to `/usr/share/munin/plugins`.
|
||||
Example:
|
||||
|
||||
```bash
|
||||
wget 'https://github.com/CometBackup/comet-munin-plugin/archive/refs/heads/main.zip'
|
||||
unzip comet-munin-plugin-main.zip
|
||||
cd comet-munin-plugin-main/comet-munin-plugin
|
||||
mv comet_* /usr/share/munin/plugins
|
||||
```
|
||||
|
||||
2. Set all the files aside from `comet_server.py` as executable and Create a symbolic link of these files to `/etc/munin/plugins`.
|
||||
```bash
|
||||
for plugin in comet_jobs_classification comet_jobs_status comet_latency comet_online_devices comet_server_history comet_uptime; do
|
||||
chmod +x "/usr/share/munin/plugins/$plugin"
|
||||
ln -s "/usr/share/munin/plugins/$plugin" "/etc/munin/plugins/$plugin"
|
||||
done
|
||||
```
|
||||
|
||||
3. Enter your Comet Server admin credentials in `comet_server.py`
|
||||
`nano /usr/share/munin/plugins/comet_server.py`
|
||||
From here enter your credentials on line 9-11 in the quotation marks:
|
||||
`COMET_SERVER_URL = "https://mycometserver.com/"` - This must include http and the trailing forward slash.
|
||||
`COMET_ADMIN_USERNAME = "adminUsername"`
|
||||
`COMET_ADMIN_PASSWORD = "adminPassword"`
|
||||
Then save and close nano.
|
||||
|
||||
4. Once done, restart Munin and Munin node `systemctl restart munin munin-node`
|
||||
|
||||
## Generated graphs
|
||||
|
||||
|Graph |Description
|
||||
|----------------------|----
|
||||
|Comet Online Devices|The total number of devices along with their version status
|
||||
|Comet Server History|The current number of users, devices, storage buckets, and boosters
|
||||
|Comet Job Status 24h|Jobs over the last 24 hours and their status
|
||||
|Comet Job Classification 24h|Jobs over the last 24 hours and their classification
|
||||
|Comet API Latency|Time taken to get the server configuration via an API call
|
||||
|Comet Uptime|Whether the server is online or offline
|
||||
|
||||
## Running example
|
||||
|
||||
[](example.png)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
print('graph_title Comet Job Classification 24h')
|
||||
print('graph_args --base 1000 -l 0 ')
|
||||
print('graph_category Comet')
|
||||
print('other.label Other')
|
||||
print('other.draw AREASTACK')
|
||||
print('other.min 0')
|
||||
print('other.colour dcdcdc')
|
||||
print('restore.label Restore')
|
||||
print('restore.draw AREASTACK')
|
||||
print('restore.min 0')
|
||||
print('restore.colour a561d3')
|
||||
print('backup.label Backup')
|
||||
print('backup.draw AREASTACK')
|
||||
print('backup.min 0')
|
||||
print('backup.colour 4d98d7')
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
classificationCount = cs.getJobsClassification()
|
||||
print("backup.value " + str(classificationCount["Backup"]))
|
||||
print("restore.value " + str(classificationCount["Restore"]))
|
||||
print("other.value " + str(classificationCount["Other"]))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
print('graph_title Comet Job Status 24h')
|
||||
print('graph_args --base 1000 -l 0 ')
|
||||
print('graph_category Comet')
|
||||
print('skipped.label Skipped')
|
||||
print('skipped.draw AREASTACK')
|
||||
print('skipped.min 0')
|
||||
print('skipped.colour a562d3')
|
||||
print('running.label Running')
|
||||
print('running.draw AREASTACK')
|
||||
print('running.min 0')
|
||||
print('running.colour b9b9b9')
|
||||
print('missed.label Missed')
|
||||
print('missed.draw AREASTACK')
|
||||
print('missed.min 0')
|
||||
print('missed.colour dcdcdc')
|
||||
print('warning.label Warning')
|
||||
print('warning.draw AREASTACK')
|
||||
print('warning.min 0')
|
||||
print('warning.colour fa9545')
|
||||
print('error.label Error')
|
||||
print('error.draw AREASTACK')
|
||||
print('error.min 0')
|
||||
print('error.colour cf4848')
|
||||
print('success.label Success')
|
||||
print('success.draw AREASTACK')
|
||||
print('success.min 0')
|
||||
print('success.colour 58b154')
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
statusCount = cs.getJobsStatus()
|
||||
print("success.value " + str(statusCount["Success"]))
|
||||
print("error.value " + str(statusCount["Error"]))
|
||||
print("warning.value " + str(statusCount["Warning"]))
|
||||
print("missed.value " + str(statusCount["Missed"]))
|
||||
print("running.value " + str(statusCount["Running"]))
|
||||
print("skipped.value " + str(statusCount["Skipped"]))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
print('graph_title Comet API Latency')
|
||||
print('graph_category Comet')
|
||||
print('apitime.label API time (ms)')
|
||||
print('apitime.colour 58b154')
|
||||
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
print("apitime.value " + str(cs.getAPITime()))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
print('graph_title Comet Online Devices')
|
||||
print('graph_args --base 1000 -l 0 ')
|
||||
print('graph_category Comet')
|
||||
print('offline.label Offline')
|
||||
print('offline.draw AREASTACK')
|
||||
print('offline.min 0')
|
||||
print('offline.colour cf4848')
|
||||
print('outdated.label Online (Outdated)')
|
||||
print('outdated.draw AREASTACK')
|
||||
print('outdated.min 0')
|
||||
print('outdated.colour fa9545')
|
||||
print('online.label Online')
|
||||
print('online.draw AREASTACK')
|
||||
print('online.min 0')
|
||||
print('online.colour 58b154')
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
print("offline.value " + str(cs.countOffline()))
|
||||
print("outdated.value " + str(cs.countOutdated()))
|
||||
print("online.value " + str(cs.countOnline()))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
import os
|
||||
import time
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
|
||||
COMET_SERVER_URL = ""
|
||||
COMET_ADMIN_USERNAME = ""
|
||||
COMET_ADMIN_PASSWORD = ""
|
||||
|
||||
class CometServer(object):
|
||||
def __init__(self, url = COMET_SERVER_URL, adminuser = COMET_ADMIN_USERNAME, adminpass = COMET_ADMIN_PASSWORD):
|
||||
self.url = url
|
||||
self.adminuser = adminuser
|
||||
self.adminpass = adminpass
|
||||
self.apiCache = dict()
|
||||
|
||||
def _get_response(self, endpoint, extraparams):
|
||||
apiRequest = urllib.request.Request(
|
||||
url = self.url + endpoint,
|
||||
data = urllib.parse.urlencode({
|
||||
"Username": self.adminuser,
|
||||
"AuthType": "Password",
|
||||
"Password": self.adminpass,
|
||||
**extraparams
|
||||
}).encode('utf-8')
|
||||
)
|
||||
return urllib.request.urlopen(apiRequest)
|
||||
|
||||
def _request(self, endpoint, extraparams):
|
||||
"""Make API request to Comet Server and parse response JSON"""
|
||||
shouldCache = (len(extraparams) == 0)
|
||||
|
||||
#runs if there are no extra parameters and the endpoint is in the dict
|
||||
if shouldCache and endpoint in self.apiCache:
|
||||
return self.apiCache[endpoint]
|
||||
|
||||
ret = None
|
||||
with self._get_response(endpoint, extraparams) as apiResponse:
|
||||
ret = json.loads(apiResponse.read())
|
||||
|
||||
#runs and caches the results if there are no parameters
|
||||
if shouldCache:
|
||||
self.apiCache[endpoint] = ret
|
||||
return ret
|
||||
|
||||
#---Online Devices
|
||||
def countOnline(self):
|
||||
#Count all online devices on the Comet Server
|
||||
version = self._request("api/v1/admin/meta/version", {})["Version"]
|
||||
count = 0
|
||||
for device in self._request("api/v1/admin/dispatcher/list-active", {}).values():
|
||||
if device["ReportedVersion"] == version:
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
def countOutdated(self):
|
||||
#Count all online but outdated devices on the Comet Server
|
||||
version = self._request("api/v1/admin/meta/version", {})["Version"]
|
||||
count = 0
|
||||
for device in self._request("api/v1/admin/dispatcher/list-active", {}).values():
|
||||
if device["ReportedVersion"] != version:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def countOffline(self):
|
||||
#Count all offline devices on the Comet Server
|
||||
count = 0
|
||||
for user in self._request("api/v1/admin/list-users-full", {}).values():
|
||||
count += len(user["Devices"])
|
||||
for device in self._request("api/v1/admin/dispatcher/list-active", {}).values():
|
||||
count -= 1
|
||||
return count
|
||||
|
||||
|
||||
#---Server History
|
||||
def countUsers(self):
|
||||
#Count all usernames on the Comet Server
|
||||
return len(self._request("api/v1/admin/list-users", {}))
|
||||
|
||||
def countBuckets(self):
|
||||
#Count all buckets on the Comet Server
|
||||
return len(self._request("api/v1/admin/storage/list-buckets", {}))
|
||||
|
||||
def countDevices(self):
|
||||
#Count all devices on the Comet Server
|
||||
count = 0
|
||||
for user in self._request("api/v1/admin/list-users-full", {}).values():
|
||||
count += len(user["Devices"])
|
||||
return count
|
||||
|
||||
def countBoosters(self):
|
||||
#Count all boosters on the Comet Server
|
||||
count = 0
|
||||
paidBoosters = {"engine1/exchangeedb", "engine1/mssql", "engine1/vsswriter", "engine1/hyperv", "engine1/windisk", "engine1/mongodb", "engine1/winmsofficemail"}
|
||||
for user in self._request("api/v1/admin/list-users-full", {}).values():
|
||||
singleCheck = set()
|
||||
hasItem = set()
|
||||
for source in user["Sources"].values():
|
||||
#This check is for the very small chance there is no Device tied to the Protected Item
|
||||
if source["OwnerDevice"] != "":
|
||||
hasItem.add(source["OwnerDevice"])
|
||||
if source["Engine"] in paidBoosters:
|
||||
singleCheck.add(source["OwnerDevice"]+source["Engine"])
|
||||
for deviceID in hasItem:
|
||||
device = user['Devices'][deviceID]
|
||||
if "PlatformVersion" in device and "distribution" in device["PlatformVersion"] and device["PlatformVersion"]["distribution"] == "Synology DSM":
|
||||
singleCheck.add(deviceID+"Synology")
|
||||
count += len(singleCheck)
|
||||
return count
|
||||
|
||||
#---JobStatus
|
||||
def getJobsStatus(self):
|
||||
#Return job status from the last 24h on the Comet Server
|
||||
today = int(time.time())
|
||||
yesterday = int(time.time() - 86400)
|
||||
statusCount = {
|
||||
"Success": 0,
|
||||
"Error": 0,
|
||||
"Warning": 0,
|
||||
"Missed": 0,
|
||||
"Running": 0,
|
||||
"Skipped": 0
|
||||
}
|
||||
for job in self._request("api/v1/admin/get-jobs-for-date-range", {"Start": yesterday, "End": today}):
|
||||
if (5000 <= job["Status"] <= 5999):
|
||||
statusCount["Success"] += 1
|
||||
elif (6000 <= job["Status"] <= 6999):
|
||||
statusCount["Running"] += 1
|
||||
elif (job["Status"] == 7001):
|
||||
statusCount["Warning"] += 1
|
||||
elif (job["Status"] == 7004):
|
||||
statusCount["Missed"] += 1
|
||||
elif (job["Status"] == 7006):
|
||||
statusCount["Skipped"] += 1
|
||||
else:
|
||||
statusCount["Error"] += 1
|
||||
return statusCount
|
||||
|
||||
#---JobClassification
|
||||
def getJobsClassification(self):
|
||||
#Return job classification from the last 24h on the Comet Server
|
||||
today = int(time.time())
|
||||
yesterday = int(time.time() - 86400)
|
||||
classificationCount = {
|
||||
"Backup": 0,
|
||||
"Restore": 0,
|
||||
"Other": 0
|
||||
}
|
||||
for job in self._request("api/v1/admin/get-jobs-for-date-range", {"Start": yesterday, "End": today}):
|
||||
if (job["Classification"] == 4001):
|
||||
classificationCount["Backup"] += 1
|
||||
elif (job["Classification"] == 4002):
|
||||
classificationCount["Restore"] += 1
|
||||
else:
|
||||
classificationCount["Other"] += 1
|
||||
return classificationCount
|
||||
|
||||
#---Latency
|
||||
def getAPITime(self):
|
||||
#Return API time to the Comet Server
|
||||
start = datetime.now()
|
||||
_ = self._get_response("api/v1/admin/meta/server-config/get", {})
|
||||
end = datetime.now()
|
||||
return (end - start).total_seconds()*1000
|
||||
|
||||
#---OnlineCheck
|
||||
def checkOnline(self):
|
||||
#Checks if the server is online
|
||||
try:
|
||||
response = self._get_response("api/v1/admin/meta/server-config/get", {})
|
||||
return 1
|
||||
except:
|
||||
return 0
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
print('graph_title Comet Server History')
|
||||
print('graph_args --base 1000')
|
||||
print('graph_category Comet')
|
||||
print('buckets.label Storage buckets')
|
||||
print('buckets.colour 4d98d7')
|
||||
print('users.label Users')
|
||||
print('users.colour cf4848')
|
||||
print('devices.label Devices')
|
||||
print('devices.colour 58b154')
|
||||
print('boosters.label Boosters')
|
||||
print('boosters.colour a561d3')
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
print("buckets.value " + str(cs.countBuckets()))
|
||||
print("users.value " + str(cs.countUsers()))
|
||||
print("devices.value " + str(cs.countDevices()))
|
||||
print("boosters.value " + str(cs.countBoosters()))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from comet_server import CometServer
|
||||
|
||||
def config():
|
||||
cs = CometServer()
|
||||
onlineCheck = cs.checkOnline()
|
||||
print('graph_title Comet Uptime')
|
||||
print('graph_args --base 1000 -l 0 ')
|
||||
print('graph_category Comet')
|
||||
print('online.label Online')
|
||||
print('online.draw AREA')
|
||||
print('online.min 0')
|
||||
if onlineCheck == 1:
|
||||
print('online.colour 58b154')
|
||||
else:
|
||||
print('online.colour cf4848')
|
||||
|
||||
def main():
|
||||
cs = CometServer()
|
||||
print("online.value " + str(cs.checkOnline()))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'config':
|
||||
config()
|
||||
else:
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 184 KiB |
Loading…
Reference in New Issue