Merge pull request #1434 from kimheino/master

network/nft_counter: add plugin to graph nftables named counters

plugins/docker/docker_

@@ -26,6 +26,7 @@ Or choose a subset of those you want.
     ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_images
     ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_memory
     ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_network
+    ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_size
     ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_status
     ln -s /usr/share/munin/plugins/docker_ /etc/munin/plugins/docker_volumes
 
@@ -285,6 +286,14 @@ def parallel_container_stats(client):
     return stats.items()
 
 
+def print_containers_size(client):
+    for container in client.api.df()['Containers']:
+        size = container['SizeRw'] if 'SizeRw' in container else 0
+        # first char of name is always / so we skip it
+        clean_container_name = clean_fieldname(container['Names'][0][1:])
+        print(clean_container_name + '.value', size)
+
+
 def print_containers_cpu(client):
     for container, stats in parallel_container_stats(client):
         cpu_percent = 0.0
@@ -430,6 +439,23 @@ def volumes(client, mode):
         print('volumes_quantity.extinfo', ', '.join(volume_summary(v) for v in client.volumes))
 
 
+def size(client, mode):
+    if mode == "config":
+        print("graph_title Docker containers size")
+        print("graph_args --base 1024 -l 0")
+        print("graph_vlabel Bytes")
+        print("graph_category virtualization")
+        print("graph_info This graph shows docker container size.")
+        print("graph_total Total container size")
+        for container in client.all_containers:
+            fieldname = clean_fieldname(container.name)
+            print("{}.label {}".format(fieldname, container.name))
+            print("{}.draw AREASTACK".format(fieldname))
+            print("{}.info {}".format(fieldname, container_attributes(container)))
+    else:
+        print_containers_size(client)
+
+
 def cpu(client, mode):
     if mode == "config":
         graphlimit = str(os.cpu_count() * 100)
@@ -501,6 +527,7 @@ def main():
         'memory',
         'network',
         'status',
+        'size',
         'volumes',
     ]
 

plugins/network/nft_counter

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+
+"""Munin plugin to monitor nftables named counters.
+
+=head1 NAME
+
+nft_counter - monitor nftables named counters
+
+=head1 APPLICABLE SYSTEMS
+
+Linux systems with nftables running and counters defined.
+
+=head1 CONFIGURATION
+
+Include/exclude regexp filters can be configured for each graph. Default
+is to include everything.
+
+Example:
+
+  [nft_counter]
+  user root
+
+  # Graph only "inet_foomuuri_ssh" and "inet_foomuuri_http" counters
+  env.bytes_include_re inet_foomuuri_(ssh|http)
+  #env.bytes_exclude_re nothing
+
+  # Graph everything but "inet_foomuuri_ssh" counter
+  #env.packets_include_re .
+  env.packets_exclude_re inet_foomuuri_ssh
+
+  # Automatically find "_in" and "_out" counters and graph them in
+  # traffic style, similar to "if_" plugin.
+  #env.traffic_include_re .
+  #env.traffic_exclude_re nothing
+
+=head1 AUTHOR
+
+Kim B. Heino <b@bbbs.net>
+
+=head1 LICENSE
+
+GPLv2
+
+=head1 MAGIC MARKERS
+
+ #%# family=auto
+ #%# capabilities=autoconf
+
+=cut
+"""
+
+import os
+import re
+import subprocess
+import json
+import sys
+import unicodedata
+
+
+def safename(name):
+    """Return safe variable name."""
+    # Convert ä->a as isalpha('ä') is true
+    value = unicodedata.normalize('NFKD', name)
+    value = value.encode('ASCII', 'ignore').decode('utf-8')
+
+    # Remove non-alphanumeric chars
+    return ''.join(char.lower() if char.isalnum() else '_' for char in value)
+
+
+def collect_data():
+    """Run nft and parse its output."""
+    # List counters in nft json
+    try:
+        text = subprocess.run(['nft', '--json', 'list', 'counters'],
+                              stdout=subprocess.PIPE, check=False,
+                              encoding='utf-8', errors='ignore').stdout
+    except FileNotFoundError:
+        return {}
+    try:
+        data = json.loads(text)
+    except ValueError:
+        return {}
+
+    # Parse counters from nft json
+    counters = {}
+    for nft_type in data.get('nftables', []):
+        if 'counter' not in nft_type:
+            continue
+        counter = nft_type['counter']
+        name = f'{counter["family"]}_{counter["table"]}_{counter["name"]}'
+        counters[safename(name)] = counter
+    return counters
+
+
+def filter_data(data, prefix):
+    """Filter collected data by config regexps."""
+    include = os.getenv(f'{prefix}_include_re', '')
+    exclude = os.getenv(f'{prefix}_exclude_re', '$^')
+    ret = {}
+    for counter, values in data.items():
+        if re.search(include, counter) and not re.search(exclude, counter):
+            ret[counter] = values
+    return ret
+
+
+def group_traffic(data):
+    """Find "_in" and "_out" pairs from counter data."""
+    counters = filter_data(data, 'traffic')
+    pairs = []
+    for name in counters:
+        if name.endswith('_in') and f'{name[:-3]}_out' in counters:
+            pairs.append(name[:-3])
+    return pairs
+
+
+def config():
+    """Print plugin config."""
+    data = collect_data()
+
+    counters = filter_data(data, 'bytes')
+    if counters:
+        print('multigraph nft_counter_bytes')
+        print('graph_title nftables counter bytes')
+        print('graph_category network')
+        print('graph_vlabel bits per ${graph_period}')
+        print('graph_args --base 1024')
+        for counter, value in counters.items():
+            if value['family'] == 'inet':
+                print(f'{counter}.label {value["name"]}')
+            else:
+                print(f'{counter}.label {value["family"]} {value["name"]}')
+            print(f'{counter}.type DERIVE')
+            print(f'{counter}.cdef {counter},8,*')
+            print(f'{counter}.min 0')
+
+    counters = filter_data(data, 'packets')
+    if counters:
+        print('multigraph nft_counter_packets')
+        print('graph_title nftables counter packets')
+        print('graph_category network')
+        print('graph_vlabel packets per ${graph_period}')
+        print('graph_args --base 1000')
+        for counter, value in counters.items():
+            if value['family'] == 'inet':
+                print(f'{counter}.label {value["name"]}')
+            else:
+                print(f'{counter}.label {value["family"]} {value["name"]}')
+            print(f'{counter}.type DERIVE')
+            print(f'{counter}.min 0')
+
+    pairs = group_traffic(data)
+    if pairs:
+        print('multigraph nft_counter_traffic')
+        print('graph_title nftables counter traffic')
+        print('graph_category network')
+        print('graph_vlabel bits in (-) / out (+) per ${graph_period}')
+        for pair in pairs:
+            print(f'{pair}_in.label received')
+            print(f'{pair}_in.type DERIVE')
+            print(f'{pair}_in.graph no')
+            print(f'{pair}_in.cdef {pair}_in,8,*')
+            print(f'{pair}_in.min 0')
+            print(f'{pair}_out.label {data[f"{pair}_in"]["name"][:-3][:15]}')
+            print(f'{pair}_out.type DERIVE')
+            print(f'{pair}_out.negative {pair}_in')
+            print(f'{pair}_out.cdef {pair}_out,8,*')
+            print(f'{pair}_out.min 0')
+
+    if os.environ.get('MUNIN_CAP_DIRTYCONFIG') == '1':
+        fetch(data)
+
+
+def fetch(data):
+    """Print values."""
+    counters = filter_data(data, 'bytes')
+    if counters:
+        print('multigraph nft_counter_bytes')
+        for counter, value in counters.items():
+            print(f'{counter}.value {value["bytes"]}')
+
+    counters = filter_data(data, 'packets')
+    if counters:
+        print('multigraph nft_counter_packets')
+        for counter, value in counters.items():
+            print(f'{counter}.value {value["packets"]}')
+
+    pairs = group_traffic(data)
+    if pairs:
+        print('multigraph nft_counter_traffic')
+        for pair in pairs:
+            print(f'{pair}_in.value {data[f"{pair}_in"]["bytes"]}')
+            print(f'{pair}_out.value {data[f"{pair}_out"]["bytes"]}')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1 and sys.argv[1] == 'autoconf':
+        print('yes' if collect_data() else 'no (no nft counters found)')
+    elif len(sys.argv) > 1 and sys.argv[1] == 'config':
+        config()
+    else:
+        fetch(collect_data())

plugins/synapse/synapse_

@@ -74,7 +74,7 @@ AUTH_TOKEN="${auth_token:-}"
 INTERVAL="${interval:-300}"
 PORT="${port:-443}"
 ADMIN_API_PATH="${admin_api_path:-/_synapse/admin}"
-QUERY_LIMIT="${query_limit:-1000}"
+QUERY_LIMIT="${query_limit:-10000}"
 HOMESERVER="${0##*synapse_}"
 SCHEME="${scheme:-https}://"
 TIMEOUT="${timeout:-2}"
@@ -175,11 +175,13 @@ REPORTS=$(fetch_url -H "Authorization: Bearer ${AUTH_TOKEN}" "${SCHEME}${HOMESER
 echo multigraph synapse_users_"${CLEANHOMESERVER}"
 if USERS="$(echo "$USERS" | jq -r)"; then
 	total="$(echo "$USERS"  | jq -r .total)"
+	puppets="$(echo "$USERS"  | grep -c '"last_seen_ts": null')"
 	bots="$(echo "$USERS"  | grep -c '"user_type": "bot"')"
-	total_registered=$(( total - bots ))
+	virtual_users=$(( puppets + bots ))
+	total_registered=$(( total - virtual_users ))
 	echo total_registered.value "$total_registered"
 	active="$(echo "$USERS"  | grep -c '"deactivated": 0')"
-	active_users=$(( active - bots ))
+	active_users=$(( active - virtual_users ))
 	echo active_users.value "$active_users"
 	echo bots.value "$bots"
 	# Convert to miliseconds
@@ -189,7 +191,7 @@ if USERS="$(echo "$USERS" | jq -r)"; then
 	last_seen_times_ms=$(echo "$USERS" | grep -E "\"last_seen_ts\": [0-9]+")
 	online="$(echo "$last_seen_times_ms" | awk -v "count=0" -F": " '$2 > "'$time_interval_ago'" {count++} END {print count}')"
 	online_users=$(( online - bots ))
-	echo online_users.value "$bots"
+	echo online_users.value "$online_users"
 	echo deactivated_users.value "$(echo "$USERS" | grep -c '"deactivated": 1')"
 	echo erased_users.value "$(echo "$USERS" | grep -c '"erased": true')"
 else
@@ -217,10 +219,9 @@ else
 	echo "rooms_total.value U"
 fi
 echo multigraph synapse_reports_"${CLEANHOMESERVER}"
-if [ "$REPORTS" -eq "$REPORTS" ]; then
-	echo event_reports.value "$REPORTS"
+if [ -n "$REPORTS" ] && [ "$REPORTS" -eq "$REPORTS" ]; then
+        echo event_reports.value "$REPORTS"
 else
     echo "event_reports.value U"
 fi
 
-