monitoring: add munin again... (#790)

Reviewed-on: #790
Co-authored-by: Michael Grote <michael.grote@posteo.de>
Co-committed-by: Michael Grote <michael.grote@posteo.de>

munin: nacharbeiten (#791)

Reviewed-on: #791
Co-authored-by: Michael Grote <michael.grote@posteo.de>
Co-committed-by: Michael Grote <michael.grote@posteo.de>

munin: remove playbook fixed (#792)

Reviewed-on: #792
Co-authored-by: Michael Grote <michael.grote@posteo.de>
Co-committed-by: Michael Grote <michael.grote@posteo.de>

docker-compose/munin/docker-compose.yml.j2

@@ -0,0 +1,42 @@
+version: '3'
+services:
+  munin:
+    container_name: "munin-master"
+    image: registry.mgrote.net/munin-server:v1.1.64
+    restart: always
+    environment:
+      MAILCONTACT: michael.grote@posteo.de
+      MAILSERVER: mail-relay
+      MAILPORT: 25
+      MAILFROM: munin@mgrote.net
+      MAILUSER: munin@mgrote.net
+      MAILNAME: Munin
+      MAILDOMAIN: mgrote.net
+      TZ: Europe/Berlin
+      CRONDELAY: 5
+      NODES: |
+        fileserver3.mgrote.net:fileserver3.mgrote.net
+        ansible2.mgrote.net:ansible2.mgrote.net
+        pve5.mgrote.net:pve5.mgrote.net
+        gitea.mgrote.net:gitea.mgrote.net
+        docker10.mgrote.net:docker10.mgrote.net
+        pbs.mgrote.net:pbs.mgrote.net
+        blocky.mgrote.net:blocky.mgrote.net
+      # z.B.
+      # computer-test.mgrote.net.test:192.68.2.4
+      # computer.mgrote.net:computer.mgrote.net
+    volumes:
+      - db:/var/lib/munin
+      - logs:/var/log/munin
+      - cache:/var/cache/munin
+    ports:
+      - 1234:80
+
+volumes:
+  db:
+  logs:
+  cache:
+
+networks:
+  mail-relay:
+    external: true

friedhof/mgrote_munin_node/readme.md

@@ -1,12 +0,0 @@
-## mgrote.munin-node
-
-### Beschreibung
-Installiert munin-node + Plugins.
-
-### getestet auf
-- [x] Ubuntu (>=18.04)
-- [ ] Debian
-- [x] ProxMox 6.1
-
-### Variablen + Defaults
-see [defaults](./defaults/main.yml)

group_vars/all.yml

@@ -84,11 +84,6 @@ ntp_chrony_logging: false
 postfix_smtp_server: docker10.mgrote.net
 postfix_smtp_server_port: 1025
 
-### mgrote_tmux
-tmux_conf_destination: "/home/mg/.tmux.conf"
-tmux_bashrc_destination: "/home/mg/.bashrc"
-tmux_standardsession_name: "default"
-
 ### mgrote_fail2ban
 f2b_bantime: 300
 f2b_findtime: 300
@@ -103,6 +98,11 @@ ufw_rules:
     protocol: tcp
     comment: 'ssh'
     from_ip: 0.0.0.0/0
+  - rule: allow
+    to_port: 4949
+    protocol: tcp
+    comment: 'munin'
+    from_ip: 192.168.2.0/24
 ufw_default_incoming_policy: deny
 ufw_default_outgoing_policy: allow
 
@@ -137,6 +137,8 @@ apt_packages_common:
   - pwgen
   - keychain
   - fwupd
+  - bc
+  - jq
 apt_packages_physical:
   - s-tui
   - smartmontools
@@ -201,6 +203,43 @@ sanoid_templates:
 ### mgrote_zfs_sanoid
 sanoid_deb_url: http://docker10.mgrote.net:3344/sanoid_3.0.4.deb
 
+### mgrote_munin_node
+munin_node_bind_host: "0.0.0.0"
+munin_node_bind_port: "4949"
+munin_node_allowed_cidrs: [192.168.2.0/24]
+munin_node_disabled_plugins:
+  - name: meminfo # zu hohe last
+  - name: hddtemp2 # ersetzt durch hddtemp_smartctl
+  - name: ntp # verursacht zu viele dns ptr request
+  - name: hddtempd # ersetzt durch hddtemp_smartctl
+  - name: squid_cache # proxmox
+  - name: squid_objectsize # proxmox
+  - name: squid_requests # proxmox
+  - name: squid_traffic # proxmox
+  - name: timesync
+munin_node_plugins:
+  - name: chrony
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+  - name: systemd_status
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: lvm_
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/disk/lvm_
+    config: |
+      [lvm_*]
+      user root
+  - name: fail2ban
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+
 # Ansible Variablen
 ### User
 ansible_user: "ansible-user"

group_vars/blocky.yml

@@ -9,6 +9,11 @@ ufw_rules:
     protocol: tcp
     comment: 'ssh'
     from_ip: 0.0.0.0/0
+  - rule: allow
+    to_port: 4949
+    protocol: tcp
+    comment: 'munin'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: 53
     comment: 'dns'
@@ -50,14 +55,14 @@ blocky_custom_lookups: # optional
     ip: 192.168.2.43
   - name: ci.mgrote.net
     ip: 192.168.2.43
-  - name: git.mgrote.net
-    ip: 192.168.2.43
   - name: miniflux.mgrote.net
     ip: 192.168.2.43
   - name: nextcloud.mgrote.net
     ip: 192.168.2.43
   - name: registry.mgrote.net
     ip: 192.168.2.43
+  - name: git.mgrote.net
+    ip: 192.168.2.43
   # Intern
   - name: ads2700w.mgrote.net
     ip: 192.168.2.147
@@ -80,3 +85,38 @@ blocky_custom_lookups: # optional
 
 ### mgrote_restic
 restic_repository: "//192.168.2.54/restic"
+
+### mgrote_munin_node
+# kann git.mgrote.net nicht auflösen, deshalb hiermit IP
+munin_node_plugins:
+  - name: chrony
+    src: http://192.168.2.44:3000/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+  - name: systemd_status
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: lvm_
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/disk/lvm_
+    config: |
+      [lvm_*]
+      user root
+  - name: fail2ban
+    src: http://192.168.2.44:3000/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+  - name: dnsresponse_192.168.2.1
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/network/dns/dnsresponse_
+  - name: dnsresponse_192.168.2.37
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/network/dns/dnsresponse_
+  - name: dnsresponse_127.0.0.1
+    src: http://192.168.2.44:3000/mg/mirror-munin-contrib/raw/branch/master/plugins/network/dns/dnsresponse_
+    config: |
+      [dnsresponse_*]
+      env.site www.heise.de
+      env.times 20

group_vars/docker.yml

@@ -18,6 +18,15 @@ pvresize_to_max: true
 ### mgrote_restic
 restic_folders_to_backup: "/ /var/lib/docker" # --one-file-system ist gesetzt, also werden weitere Dateisysteme nicht eingeschlossen, es sei denn sie werden hier explizit angegeben; https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files
 
+### geerlingguy.pip
+pip_package: python3-pip
+pip_install_packages:
+  - name: docker # für munin-plugin docker_
+
+### mgrote.apt_manage_packages
+apt_packages_extra:
+  - libnet-dns-perl # für munin: dnsresponse_*
+
 ### mgrote_user
 users:
   - username: mg
@@ -71,3 +80,63 @@ repos_override: # mit docker-repos
 
 ### mgrote_systemd_resolved
 systemd_resolved_nameserver: 192.168.2.37
+
+### mgrote_munin_node
+munin_node_allowed_cidrs: [0.0.0.0/0] # weil der munin-server aus einem anderen subnet zugreift
+munin_node_plugins:
+  - name: systemd_status
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: lvm_
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/disk/lvm_
+    config: |
+      [lvm_*]
+      user root
+  - name: fail2ban
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+  - name: docker_containers
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/docker/docker_
+    config: |
+      [docker_*]
+      user root
+      env.DOCKER_HOST unix://run/docker.sock
+  - name: docker_cpu
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/docker/docker_
+  - name: docker_memory
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/docker/docker_
+  - name: docker_network
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/docker/docker_
+  - name: docker_volumes
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/docker/docker_
+  - name: docker_volumesize
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/docker/docker_volumesize
+  - name: chrony
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+
+### oefenweb.ufw
+ufw_rules:
+  - rule: allow
+    to_port: 22
+    protocol: tcp
+    comment: 'ssh'
+    from_ip: 0.0.0.0/0
+  - rule: allow
+    to_port: 4949
+    protocol: tcp
+    comment: 'munin'
+    from_ip: 192.168.2.0/24
+  - rule: allow
+    from_ip: 192.168.0.0/16
+    comment: 'docker networks'
+  - rule: allow
+    from_ip: 172.0.0.0/8
+    comment: 'docker networks'

group_vars/fileserver.yml

@@ -9,6 +9,11 @@ ufw_rules:
     protocol: tcp
     comment: 'ssh'
     from_ip: 0.0.0.0/0
+  - rule: allow
+    to_port: 4949
+    protocol: tcp
+    comment: 'munin'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: 445
     comment: 'smb'
@@ -25,3 +30,30 @@ smb_client_min_protocol: "SMB2"
 smb_client_max_protocol: "SMB3_11"
 smb_enable_snapshots_dir: true
 smb_enable_snapshots_shadow: true
+
+### mgrote_munin_node
+munin_node_plugins:
+  - name: chrony
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+  - name: systemd_status
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: fail2ban
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+  - name: samba
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/samba
+    config: |
+      [samba]
+      user root
+      group root
+      env.smbstatus /usr/bin/smbstatus
+      env.ignoreipcshare 1

group_vars/gitea.yml

@@ -29,6 +29,11 @@ ufw_rules:
     protocol: tcp
     comment: 'ssh'
     from_ip: 0.0.0.0/0
+  - rule: allow
+    to_port: 4949
+    protocol: tcp
+    comment: 'munin'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: "{{ gitea_http_port }}"
     protocol: tcp

group_vars/pbs.yml

@@ -33,3 +33,55 @@ users:
     public_ssh_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJcBwOjanQV6sFWaTetqpl20SVe3aRzGjKbsp7hKkDCE mg@irantu
     allow_sudo: true
     allow_passwordless_sudo: true
+
+### mgrote_munin_node
+munin_node_plugins:
+  - name: chrony
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+  - name: systemd_status
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: fail2ban
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+  - name: zfs_arcstats
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_arcstats
+  - name: zfsonlinux_stats_
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfsonlinux_stats_
+  - name: zpool_iostat
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zpool_iostat
+  - name: zfs_list
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_list
+    config: |
+      [zfs_list]
+      env.ignore_datasets_pattern autodaily
+  - name: zfs_count
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_pool_dataset_count
+  - name: zpool_iostat
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zpool_iostat
+  - name: zpool_capacity
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zpool_capacity
+munin_node_disabled_plugins:
+  - meminfo # zu hohe last
+  - hddtemp2 # ersetzt durch hddtemp_smartctl
+  - ntp # verursacht zu viele dns ptr request
+  - hddtempd # ersetzt durch hddtemp_smartctl
+  - squid_cache # proxmox
+  - squid_objectsize # proxmox
+  - squid_requests # proxmox
+  - squid_traffic # proxmox
+  - lvm_
+  - timesync
+  - lxc_guests
+munin_node_allowed_cidrs:
+  - 192.168.3.0/24
+  - 192.168.2.0/24
+...

group_vars/pve.yml

@@ -39,6 +39,73 @@ apt_packages_extra:
   - open-vm-tools
   - systemd-boot
 
+### mgrote_munin_node
+munin_node_plugins:
+  - name: chrony
+    src: https://git.mgrote.net/Mirror/munin-contrib/raw/branch/master/plugins/chrony/chrony
+  - name: systemd_status
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_status
+  - name: systemd_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/systemd/systemd_mem
+    config: |
+      [systemd_mem]
+      env.all_services true
+  - name: fail2ban
+    src: https://git.mgrote.net/mg/munin-plugins/raw/branch/master/extern/fail2ban
+    config: |
+      [fail2ban]
+      env.client /usr/bin/fail2ban-client
+      env.config_dir /etc/fail2ban
+      user root
+  - name: zfs_arcstats
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_arcstats
+  - name: zfsonlinux_stats_
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfsonlinux_stats_
+  - name: zpool_iostat
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zpool_iostat
+  - name: zfs_list
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_list
+    config: |
+      [zfs_list]
+      env.ignore_datasets_pattern autodaily
+  - name: zpool_capacity
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zpool_capacity
+  - name: kvm_mem
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/libvirt/kvm_mem
+  - name: kvm_net
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/libvirt/kvm_net
+  - name: kvm_io
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/libvirt/kvm_io
+    config: |
+      [kvm_io]
+      user root
+  - name: kvm_cpu
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/libvirt/kvm_cpu
+  - name: proxmox_count
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/proxmox/proxmox_vm_count
+    config: |
+      [proxmox_count]
+      user root
+      group root
+  - name: zfs_count
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/zfs/zfs_pool_dataset_count
+  - name: ksm_
+    src: https://git.mgrote.net/mg/mirror-munin-contrib/raw/branch/master/plugins/system/kernel_same_page_merging
+munin_node_disabled_plugins:
+  - meminfo # zu hohe last
+  - hddtemp2 # ersetzt durch hddtemp_smartctl
+  - ntp # verursacht zu viele dns ptr request
+  - hddtempd # ersetzt durch hddtemp_smartctl
+  - squid_cache # proxmox
+  - squid_objectsize # proxmox
+  - squid_requests # proxmox
+  - squid_traffic # proxmox
+  - lvm_
+  - slab
+  - timesync
+  - lxc_guests
+
 # Ansible Variablen
 ### sudo
 sudo: false
+...

playbooks/2_all.yml

@@ -3,3 +3,4 @@
 - ansible.builtin.import_playbook: base/system.yml
 - ansible.builtin.import_playbook: base/users.yml
 - ansible.builtin.import_playbook: base/ufw.yml
+- ansible.builtin.import_playbook: base/monitoring.yml

playbooks/base/monitoring.yml

@@ -0,0 +1,11 @@
+---
+- hosts: all
+  roles:
+    - role: mgrote_munin_node
+      become: true
+      tags: "munin"
+      when: "not 'laptop' in group_names"
+
+  ### Die Host müssen auch beim Docker-Container: "munin-master eingetragen" werden.
+  ### wird nur auf physischen Rechnern ausgeführt.
+  ### Wenn ein Plugin nicht geht: munin-node-configure --shell --families=contrib,auto | sh -x

playbooks/on-off/remove_munin_node.yml

@@ -0,0 +1,22 @@
+---
+- hosts: all
+  become: yes
+  tasks:
+    - name: Ensure packages are absent
+      become: yes
+      ansible.builtin.apt:
+        autoremove: yes
+        autoclean: yes
+        purge: yes
+        name:
+          - munin-node
+        state: absent
+
+    - name: Ensure directories are absent
+      become: yes
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - /usr/share/munin
+        - /etc/munin

roles/mgrote_munin_node/handlers/main.yml

@@ -10,6 +10,6 @@
   changed_when: "output_conf.rc != 0"
   ignore_errors: true # ignoriere fehler
 
-- name: munin-node-configure --shell - 2 # noqa ignore-errors
+- name: munin-node-configure --shell - 2 # noqa ignore-errors no-changed-when
   ansible.builtin.command: munin-node-configure --shell --families=contrib,auto | sh -x
   ignore_errors: true # ignoriere fehler

roles/mgrote_munin_node/tasks/install.yml

@@ -13,7 +13,7 @@
     state: directory
     owner: root
     group: root
-    mode: "0644"
+    mode: "0755"
   loop:
     - /etc/munin
     - /etc/munin/plugin-conf.d
@@ -25,5 +25,5 @@
     dest: /etc/munin/munin-node.conf
     owner: root
     group: root
-    mode: "0644"
+    mode: "0755"
   notify: restart munin-node

roles/mgrote_munin_node/tasks/remove.yml

@@ -1,7 +1,7 @@
 ---
 - name: remove unwanted plugins
   ansible.builtin.file:
-    path: "{{ munin_plugin_dest_path }}{{ item.name }}"
+    path: "{{ munin_plugin_dest_path }}{{ item }}"
     state: absent
   loop: "{{ munin_node_disabled_plugins }}"
   notify: restart munin-node
@@ -10,7 +10,7 @@
 - name: remove additional plugin-config
   ansible.builtin.file:
     state: absent
-    dest: "{{ munin_plugin_conf_dest_path }}{{ item.name }}"
+    dest: "{{ munin_plugin_conf_dest_path }}{{ item }}"
   notify: restart munin-node
   loop: "{{ munin_node_disabled_plugins }}"
   when: munin_node_disabled_plugins is defined