diff --git a/plugins/shorewall/shorewall_log b/plugins/shorewall/shorewall_log new file mode 100755 index 00000000..56a77ec1 --- /dev/null +++ b/plugins/shorewall/shorewall_log @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +# -*- python -*- + +""" + +=head1 NAME + +Plugin to monitor iptables logs configured by shorewall + +=head1 CONFIGURATION + + logfile: Path to the iptables log file, or "journald" to use journald. + When journalctl exists, the default is "journald", otherwise + "/var/log/kern.log". + journalctlargs: Arguments passed to journalctl to select the right logs. + The default is "SYSLOG_IDENTIFIER=kernel". + taggroups: Space separated list of groups. A group contains a tag if the + group is substring of the tag. Tags belonging to the same group + will be combined in one graph. + tagfilter: Space separated list of filters. When a tag is matched by a + filter (i.e. if the filter is a substring of the tag) it is + ignored. + prefixformat: The format of the prefix configured in iptables, this is the + LOGFORMAT option in shorewall.conf. When not set the entire + prefix is used. + include_ungrouped: when a tag is found that does not belong to a group, + make it it's own group + +Example: + +Using /var/log/kern.log as logfile: + +=over 2 + + [shorewall_log] + group adm + env.logfile /var/log/kern.log + +=back + +Using journald: + +=over 2 + + [shorewall_log] + group systemd-journal + +=back + +=head1 HISTORY + +2017-11-03: v1.0 Bert Van de Poel : created +2020-07-16: v2.0 Vincent Vanlaer : rewrite + - read all tags from iptables config, instead of the last 24h + of logs + - add support for journald + - use cursors for accuracy + - convert to multigraph to reduce load + +=head1 USAGE + +Parameters understood: + + config (required) + autoconf (optional - used by munin-config) + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=cut +""" + + +import sys +import os + + +def autoconf() -> str: + if sys.version_info < (3, 5): + return 'no (This plugin requires python 3.5 or higher)' + + if os.getenv('MUNIN_CAP_MULTIGRAPH', '0') != '1': + return 'no (No multigraph support)' + + import shutil + + if not shutil.which('shorewall'): + return 'no (No shorewall executable found)' + + if not shutil.which('iptables-save'): + return 'no (No iptables-save executable found, required for tag enumeration)' + + return 'yes' + + +if len(sys.argv) == 2 and sys.argv[1] == "autoconf": + print(autoconf()) + sys.exit() + + +from collections import defaultdict, namedtuple +from typing import Set, Iterator, TextIO, Dict, Tuple +from itertools import takewhile +from subprocess import run, PIPE +import shlex +import shutil +import pickle +import re + + +logfile = os.getenv('logfile', 'journald' if shutil.which('journalctl') else '/var/log/kern.log') +journalctl_args = list(shlex.split(os.getenv('journalctlargs', + 'SYSLOG_IDENTIFIER=kernel'))) +taggroups = os.getenv('taggroups') +taggroups = taggroups.split() if taggroups else [] +tagfilter = os.getenv('tagfilter') +tagfilter = tagfilter.split() if tagfilter else [] +include_ungrouped = (os.getenv('includeungrouped', 'true').lower() == 'true') +prefix_format = os.getenv('prefixformat') +if prefix_format: + if sys.version_info < (3, 7): + prefix_format = re.escape(prefix_format).replace('\\%s', '(.+)').replace('\\%d', '\\d+') + else: # % is no longer escaped + prefix_format = re.escape(prefix_format).replace('%s', '(.+)').replace('%d', '\\d+') + prefix_format = re.compile(prefix_format) + + +def get_logtags() -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]: + rules = (run(['iptables-save'], stdout=PIPE, universal_newlines=True). + stdout.splitlines()) + tags = defaultdict(set) + groups = defaultdict(set) + + # every line is an iptables rule, in the iptables command/args syntax + # (without the 'iptables' command listed), eg. + # "-A INPUT -p tcp -s 10.3.3.7 -j DROP" + for line in rules: + args = iter(shlex.split(line)) + for arg in args: + # we only want rules that log packets, not that accept/drop/... + if arg == '-j' and next(args) != 'LOG': + break + # and we only need to know the logging tag, and add it to the list + if arg == '--log-prefix': + prefix = next(args) + + if prefix_format: + tag = prefix_format.match(prefix)[1] + else: + tag = prefix.rstrip() + + if any(ignored in tag for ignored in tagfilter): + continue + tags[tag].add(prefix) + + for group in taggroups: + if group in tag: + groups[group].add(tag) + break + else: + if include_ungrouped: + groups[tag].add(tag) + + break + + return groups, tags + + +State = namedtuple('State', ['journal', 'file']) + + +def load_state() -> State: + try: + with open(os.getenv('MUNIN_STATEFILE'), 'rb') as f: + return pickle.load(f) + except OSError: + return State(None, None) + + +def save_state(state: State): + with open(os.getenv('MUNIN_STATEFILE'), 'wb') as f: + return pickle.dump(state, f) + + +def get_lines_journalctl(state: State) -> Iterator[str]: + cursor = state.journal + + def catch_cursor(line: str): + cursor_id = '-- cursor: ' + if line.startswith(cursor_id): + save_state(State(line[len(cursor_id):], None)) + return False + else: + return True + + if not cursor: # prevent reading the entire journal on first run + journal = run(['journalctl', '--no-pager', '--quiet', '--lines=0', + '--show-cursor', *journalctl_args], + stdout=PIPE, universal_newlines=True) + else: + journal = run(['journalctl', '--no-pager', '--quiet', '--show-cursor', + '--after-cursor', cursor, *journalctl_args], + stdout=PIPE, universal_newlines=True) + + yield from filter(catch_cursor, journal.stdout.splitlines()) + + +def reverse_read(f: TextIO) -> Iterator[str]: + BUFSIZE = 4096 + f.seek(0, 2) + position = f.tell() + remainder = '' + while position > 0: + position = max(position - BUFSIZE, 0) + f.seek(position) + lines = f.read(BUFSIZE).splitlines() + lines[-1] += remainder + remainder = lines.pop(0) + yield from reversed(lines) + yield remainder + + +def get_lines_logfile(path: str, state: State) -> Iterator[str]: + with open(path, 'r') as f: + cursor = state.file + + reader = reverse_read(f) + + if not cursor: + save_state(State(None, next(reader))) + return + else: + new_cursor = next(reader) + save_state(State(None, new_cursor)) + yield new_cursor + yield from takewhile(lambda x: x != cursor, reader) + + +def get_tagcount(tags: Dict[str, Set[str]]) -> Dict[str, int]: + count = defaultdict(int) + state = load_state() + + if logfile == 'journald': + lines = get_lines_journalctl(state) + offset = 5 + else: + lines = get_lines_logfile(logfile, state) + offset = 6 + + for line in lines: + if 'IN=' not in line: + continue + line = line.replace(' ', ' ').split(' ', maxsplit=offset)[-1] + + for tag, prefixes in tags.items(): + for prefix in prefixes: + if line.startswith(prefix): + count[tag] += 1 + break + else: + continue + break + + return count + + +def fetch(): + groups, logtags = get_logtags() + tagcount = get_tagcount(logtags) + + for group, tags in groups.items(): + print('multigraph shorewall_{}'.format(group)) + for tag in tags: + print('{}.value {}'.format(tag.lower(), tagcount[tag])) + + +def config(): + + for group, tags in get_logtags()[0].items(): + print('multigraph shorewall_{}'.format(group)) + print('graph_title Shorewall Logs for {}'.format(group)) + print('graph_vlabel entries per ${graph_period}') + print('graph_category shorewall') + + for tag in sorted(tags): + print('{}.label {}'.format(tag.lower(), tag)) + print('{}.type ABSOLUTE'.format(tag.lower())) + print('{}.draw AREASTACK'.format(tag.lower())) + + +if len(sys.argv) == 2 and sys.argv[1] == "config": + config() +else: + fetch() + +# flake8: noqa: E265,E402