docker-compose/loki/datasources.yaml.j2

@@ -0,0 +1,31 @@
+# Configuration file version
+apiVersion: 1
+
+# Mark provisioned data sources for deletion if they are no longer in a provisioning file.
+# It takes no effect if data sources are already listed in the deleteDatasources section.
+prune: true
+
+# List of data sources to insert/update depending on what's
+# available in the database.
+datasources:
+  # <string, required> Sets the name you use to refer to
+  # the data source in panels and queries.
+  - name: Loki
+    # <string, required> Sets the data source type.
+    type: loki
+    # <string, required> Sets the access mode, either
+    # proxy or direct (Server or Browser in the UI).
+    # Some data sources are incompatible with any setting
+    # but proxy (Server).
+    access: direct
+    # <int> Sets the organization id. Defaults to orgId 1.
+    orgId: 1
+    # <string> Sets the data source's URL, including the
+    # port.
+    url: http://loki:3100
+    # <int> Sets the version. Used to compare versions when
+    # updating. Ignored when creating a new data source.
+    version: 1
+    # <bool> Allows users to edit data sources from the
+    # Grafana UI.
+    editable: false

docker-compose/loki/docker-compose.yml.j2

@@ -0,0 +1,50 @@
+# https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/prometheus_grafana_loki
+# https://ghazanfaralidevops.medium.com/grafana-loki-promtail-complete-end-to-end-project-d698aaa636d6
+services:
+  loki:
+    image: grafana/loki:3.2.0
+    container_name: loki
+    restart: always
+    volumes:
+      - loki:/loki
+      - ./loki-config.yml:/etc/loki-config.yml
+    command:
+      - '-config.file=/etc/loki-config.yml'
+    ports:
+      - "3100:3100"
+    networks:
+      - internal
+
+  grafana:
+    image: grafana/grafana:9.4.3
+    container_name: grafana
+    restart: always
+    env_file: grafana.env
+    volumes:
+      - grafana:/var/lib/grafana
+      - ./datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    ports:
+      - "3101:3000"
+    networks:
+      - internal
+      - postfix
+
+######## Networks ########
+networks:
+  postfix:
+    external: true
+  internal:
+
+######## Volumes ########
+volumes:
+  loki:
+  grafana:
+
+
+# TODO: http://vm-test-2204.mgrote.net:9080/targets + wenn firewall aus
+# TODO: grafana dashboard und datasource als code? +  https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources
+# TODO: URLS: http://docker10.mgrote.net:3101/explore?orgId=1&left=%7B%22datasource%22:%22U4h5CxgHz%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22U4h5CxgHz%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D + http://docker10.mgrote.net:3100/ready + http://vm-test-2204.mgrote.net:9080/config
+# TODO: grafana an LDAP: https://github.com/lldap/lldap/blob/main/example_configs/grafana_ldap_config.toml
+# TODO: send mail testen
+# TODO: depends on + healhtchecks
+# TODO Mikrotik in Logging? oder lieber alles rsyslog

docker-compose/loki/grafana.env.j2

@@ -0,0 +1,13 @@
+# GENERAL
+TZ=Europe/Berlin
+
+# GRAFANA
+GF_SECURITY_ADMIN_USER=admin # TODO
+GF_SECURITY_ADMIN_PASSWORD=admin # TODO
+GF_USERS_ALLOW_SIGN_UP=false
+
+# GRAFANA EMAIL
+GF_SMTP_ENABLED=true
+GF_SMTP_HOST=postfix
+GF_SMTP_USER=grafana@mgrote.net
+GF_SMTP_PASSWORD=""

docker-compose/loki/loki-config.yml.j2

@@ -0,0 +1,43 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+analytics:
+  reporting_enabled: false
+
+
+# TODO alertmanager
+# TODO promtail
+# TODO prometheus
+# TODO retention
+# TODO Doku wie alles zusammenspielt

group_vars/all.yml

@@ -9,7 +9,22 @@ file_header: |
   #----------------------------------------------------------------#
 # für Zugriff auf nicht öffentliche git.mgrote.net-Repos
 ansible_forgejo_user: svc_ansible
-ansible_forgejo_user_pass: "{{ lookup('viczem.keepass.keepass', 'user_setup_forgejo_user_pass', 'password') }}" # user ist dem Repo als "Collaborator" + "RO" hinzugefügt worden
+ansible_forgejo_user_pass: "{{ lookup('viczem.keepass.keepass', 'user_setup_forgejo_user_pass', 'password') }}" # User ist dem Repo als "Collaborator" + "RO" hinzugefügt worden
+
+### patrickjahns.promtail
+promtail_version: "3.0.0" # TODO renovate
+promtail_loki_server_url: "http://docker10.mgrote.net:3100" # TODO passt das für blocky?
+promtail_user_additional_groups: root
+promtail_config_scrape_configs:
+  - job_name: system
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        host: "{{ ansible_fqdn }}"
+        job: varlogs
+        __path__: /var/log/**/**
+# TODO add docker und lokale logs wenn sie cniht in /var/log liegen: https://ruanbekker.medium.com/logging-with-docker-promtail-and-grafana-loki-d920fd790ca8
 
 ### mgrote_user_setup
 dotfiles:
@@ -83,6 +98,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
 ufw_default_incoming_policy: deny
 ufw_default_outgoing_policy: allow
 

group_vars/blocky.yml

@@ -14,6 +14,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: 53
     comment: 'dns'

group_vars/docker.yml

@@ -140,6 +140,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
   - rule: allow
     from_ip: 192.168.0.0/16
     comment: 'docker networks'

group_vars/fileserver.yml

@@ -14,6 +14,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: 445
     comment: 'smb'

group_vars/git.yml

@@ -41,6 +41,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: "{{ gitea_http_port }}"
     protocol: tcp

group_vars/ldap.yml

@@ -18,6 +18,11 @@ ufw_rules:
     protocol: tcp
     comment: 'munin'
     from_ip: 192.168.2.0/24
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
   - rule: allow
     to_port: "{{ lldap_http_port }}"
     protocol: tcp

group_vars/munin.yml

@@ -7,10 +7,14 @@ ufw_rules:
     comment: 'ssh'
     from_ip: 0.0.0.0/0
   - rule: allow
-    to_port: 80 #TODO passt?
+    to_port: 80
     protocol: tcp
     comment: 'munin'
-
+  - rule: allow
+    to_port: 9080
+    protocol: tcp
+    comment: 'promtail'
+    from_ip: 192.168.2.0/24
 ### mgrote_restic
 restic_folders_to_backup: "/usr/local /etc /root /home /var/lib/munin"
 

host_vars/docker10.mgrote.net.yml

@@ -66,6 +66,9 @@ compose_files:
     state: present
   - name: act-runner
     state: present
+  - name: loki
+    state: present
+    network: postfix
 
 ### oefenweb.ufw
 ufw_rules:

playbooks/base/monitoring.yml

@@ -4,8 +4,6 @@
     - role: mgrote_munin_node
       become: true
       tags: "munin"
-      when: "not 'laptop' in group_names"
+    - role: patrickjahns.promtail
-
+      become: true
-  ### Die Host müssen auch beim Docker-Container: "munin-master eingetragen" werden.
+      tags: "promtail"
-  ### wird nur auf physischen Rechnern ausgeführt.
-  ### Wenn ein Plugin nicht geht: munin-node-configure --shell --families=contrib,auto | sh -x