firdhof

friedhof/mgrote_zfs_sanoid_mit_syncoid/README.md

@@ -0,0 +1,85 @@
+## mgrote.zfs_sanoid
+
+### Beschreibung
+Installiert und konfiguriert ``sanoid`` + ``syncoid``.
+
+Es gibt 3 Funktionen:
+
+1. Snapshots erstellen und entfernen
+2. Snapshots senden
+3. Snapshots empfangen
+
+### getestet auf
+- ProxMox 7.*
+- Ubuntu 20.04
+
+### Variablen + Defaults
+- see [defaults](./defaults/main.yml)
+
+
+### Beispiel Playbook
+
+```yaml
+---
+- hosts: host1,host2
+  roles:
+    - { role: mgrote_zfs_sanoid, tags: "sanoid" }
+```
+
+### Beispiel - Snapshots erstellen
+
+
+#### Variablen
+
+```yaml
+---
+  sanoid_snaps_enable: true
+  sanoid_datasets:
+    - path: 'hdd_data/videos'
+      template: '31tage'
+      recursive: 'yes'
+      snapshots: true
+  sanoid_templates:
+    - name: '31tage'
+      keep_hourly: '24'                        # Aufheben (Stunde)
+      keep_daily: '31'                         # Aufheben (Tage)
+      keep_monthly: '3'                        # Aufheben (Monate)
+      keep_yearly: '0'                         # Aufheben (Jahre)
+      frequently: '16'                         # Aufheben (Minuten)
+      frequent_period: '15'                    # Intervall (alle 5 Minuten)
+      autosnap: 'yes'                          # Automatisches erstellen von Snapshots
+      autoprune: 'yes'
+
+```
+
+### Beispiel - Snapshots senden und empfangen
+
+- Host 1 = Source
+- Host 2 = Destination
+
+
+
+#### Variablen - Host 1
+
+```yaml
+  sanoid_syncoid_source_host: true
+  sanoid_syncoid_ssh_pubkey: |
+    ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3U37DGPRPDLlgxZcM0Zj/x6RVZxs7hcWBYfPywujH4+mjbpzJckr2tx3QLfxsCCjQVb4LNSEB0xsOvzDjfDsaPuG4wzqFVyZOtjI4iWg/it4ARndun33r+xSlWc5JKHH9GRK8SBOd4lXv5ylENdhWQ7z5ZF/FtCysb1JHTTYlobgXfTZ4NswJj6BBk669l13uL6zSXq6x6vm1GWiFIcIYqwM5WGSGHFoD2RNn0TJKI9A3AULPloMzWeHG3fJhoVfNY6ZB0kqpTHGoAmJUURkBFki1cJkzx3tyto4VpTzZmUyYg+qqIWbv7Me3YVJCln8JYD10uDb2oPRx6G3C9DlnzRmAVVbqCHzwvOY0H5TLTW7AXCHHgSdHaRym4oTUY9dDS/XFU3rHgexerBbi3sy1Tm0/dEU3cZFm4YOJXY/l4TeTRlhg2VbctsWE1BN1CZcoJRR+qNdJzM7Vl70Y6RGU92Y1rzSpooYVuyCFDrEIp0hAHidb5rs4paCvoxtVqak+LK8dcq0IbWxcxomEimeRG4+Opd3vo+U6subp5jqkOY0uYkFVJXaMHkP5ZIxlCFgif2A3YAPhz9IczRJaaNY3pbVgU7ybOBp+S8KRK8Ysk6OP5ApOTQVTlRhYeNqo7mpuW6139VRY5luekSCy3ehHCI9/MObhu2juF1Nz0HMeMQ== mg@irantu
+```
+
+
+#### Variablen - Host 2
+
+```yaml
+  sanoid_syncoid_timer: '*:*'
+  sanoid_syncoid_bwlimit: 30m
+  sanoid_syncoid_datasets_sync:
+    - source_host: host1.lan
+      source_dataset: hdd_data_mirror
+      destination_mount_check: hdd_data/encrypted # Wenn dieses Dataset nicht gemountet ist(z.B. durch Verschlüsselung, dann bricht syncoid ab)
+      destination_dataset: hdd_data/encrypted/syncoid/zfs1
+      skip_parent: false
+  sanoid_syncoid_ssh_privkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_private_key', 'notes') }}"
+  sanoid_syncoid_destination_host: true
+
+```

friedhof/mgrote_zfs_sanoid_mit_syncoid/defaults/main.yml

@@ -0,0 +1,49 @@
+---
+### when should sanoid be run (every 5 minutes)
+sanoid_timer: '*-*-* *:00/5'
+### when should syncoid be run
+sanoid_syncoid_timer: '*-*-* *:00:00'
+
+#  ### "Default" Datasets
+#  sanoid_datasets: # dictionary
+#    - path: 'hdd_data/data' # path to dataset; without leading /
+#      template: 'fiveminutes' # name
+#      recursive: 'no' # recursive snapshotting
+#      snapshots: true # (de)activate; can be used to disable snapshotting of subdatasets if recursive is set
+#    - path: 'hdd_data/test'
+#      snapshots: false # deaktiviert sanoid für das dataset
+#
+#  ### Templates
+#  sanoid_templates:
+#    - name: 'fiveminutes'
+#      keep_hourly: '24'                        # Aufheben (Stunde)
+#      keep_daily: '31'                         # Aufheben (Tage)
+#      keep_monthly: '6'                        # Aufheben (Monate)
+#      keep_yearly: '0'                         # Aufheben (Jahre)
+#      frequently: '36'                         # Aufheben (Minuten)
+#      frequent_period: '5'                     # Intervall (alle 5 Minuten)
+#      autosnap: 'yes'                          # Automatisches erstellen von Snapshots
+#      autoprune: 'yes'
+
+### user and group for sanoid
+sanoid_user: sanoid
+sanoid_user_group: sanoid
+
+### enable/disable features
+## enable snapshotting
+# sanoid_snaps_enable: true
+## enable sending snaps
+# sanoid_syncoid_source_host: true
+## enable receiving snaps
+# sanoid_syncoid_destination_host: true
+
+# syncoid
+#sanoid_syncoid_ssh_privkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_private_key', 'notes') }}"
+#sanoid_syncoid_ssh_pubkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_public_key', 'notes') }}"
+
+### mgrote_sanoid
+#sanoid_syncoid_datasets_sync:
+#  - source_host: pve5.mgrote.net
+#    source_dataset: hdd_data/tmp
+#    destination_mount_check: hdd_data/tmp  # zielpool
+#    destination_dataset: backup/pve5/tmp

friedhof/mgrote_zfs_sanoid_mit_syncoid/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: systemctl daemon-reload
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true

friedhof/mgrote_zfs_sanoid_mit_syncoid/tasks/destination.yml

@@ -0,0 +1,85 @@
+---
+- name: template ssh private key
+  become: true
+  ansible.builtin.template:
+    src: private_key.j2
+    dest: "/etc/sanoid/.ssh/id_sanoid"
+    owner: "{{ sanoid_user }}"
+    group: "{{ sanoid_user_group }}"
+    mode: "0400"
+  no_log: true
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: Ensure user is added to sudoers
+  become: true
+  community.general.sudoers:
+    name: "users-sudo-{{ sanoid_user }}"
+    state: present
+    user: "{{ sanoid_user }}"
+    commands: ALL
+    nopassword: true
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: template syncoid.service
+  become: true
+  ansible.builtin.template:
+    src: "syncoid.service.j2"
+    dest: /etc/systemd/system/syncoid.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - systemctl daemon-reload
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: template syncoid.sh
+  become: true
+  ansible.builtin.template:
+    src: "syncoid.sh.j2"
+    dest: /usr/bin/syncoid.sh
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: template syncoid_mail.service
+  become: true
+  ansible.builtin.template:
+    src: "syncoid_mail.service.j2"
+    dest: /etc/systemd/system/syncoid_mail.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - systemctl daemon-reload
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: template syncoid.timer
+  become: true
+  ansible.builtin.template:
+    src: "syncoid.timer.j2"
+    dest: "/etc/systemd/system/syncoid.timer"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - systemctl daemon-reload
+  when:
+    - sanoid_syncoid_destination_host
+
+- name: enable syncoid.timer
+  become: true
+  ansible.builtin.systemd:
+    name: "syncoid.timer"
+    enabled: true
+    masked: false
+    state: started
+  notify:
+    - systemctl daemon-reload
+  when:
+    - sanoid_syncoid_destination_host

friedhof/mgrote_zfs_sanoid_mit_syncoid/tasks/main.yml

@@ -0,0 +1,78 @@
+---
+- name: include user tasks
+  ansible.builtin.include_tasks: user.yml
+
+- name: install packages from repo
+  become: true
+  ansible.builtin.apt:
+    name:
+      - mbuffer
+      - lzop
+      - libcapture-tiny-perl
+      - pv
+      - libconfig-ini-perl
+      - sanoid
+    state: present
+
+- name: Overwrite syncoid script from package
+  become: true
+  ansible.builtin.get_url:
+    url: https://raw.githubusercontent.com/jimsalterjrs/sanoid/master/syncoid
+    dest: /usr/bin/syncoid
+    mode: '0755'
+    owner: root
+    group: root
+    force: true
+
+- name: create sanoid directories
+  become: true
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ sanoid_user }}"
+    group: "{{ sanoid_user_group }}"
+    mode: "0700"
+  with_items:
+    - "/etc/sanoid"
+    - "/etc/sanoid/.ssh"
+
+- name: include snaps tasks
+  ansible.builtin.include_tasks: snaps.yml
+  when:
+    - sanoid_datasets is defined
+    - sanoid_templates is defined
+    - sanoid_snaps_enable is defined
+    - sanoid_snaps_enable
+
+- name: ensure timers are enabled
+  become: true
+  ansible.builtin.systemd:
+    state: started
+    name: "{{ item }}"
+    daemon_reload: true
+    masked: false
+    enabled: true
+  loop:
+    - sanoid.timer
+
+- name: ensure services are enabled
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    masked: false
+    enabled: true
+  loop:
+    - sanoid.service
+    - sanoid-prune.service
+
+- name: include source-host tasks
+  ansible.builtin.include_tasks: source.yml
+  when:
+    - sanoid_syncoid_source_host is defined and sanoid_syncoid_source_host is true
+    - sanoid_syncoid_ssh_pubkey is defined
+
+- name: include destination-host tasks
+  ansible.builtin.include_tasks: destination.yml
+  when:
+    - sanoid_syncoid_destination_host is defined and sanoid_syncoid_destination_host is true
+    - sanoid_syncoid_ssh_privkey is defined

friedhof/mgrote_zfs_sanoid_mit_syncoid/tasks/snaps.yml

@@ -0,0 +1,46 @@
+---
+- name: Generate Sanoid Configuration
+  become: true
+  ansible.builtin.template:
+    src: sanoid.conf.j2
+    dest: "/etc/sanoid/sanoid.conf"
+    owner: "{{ sanoid_user }}"
+    group: "{{ sanoid_user_group }}"
+    mode: "0400"
+
+- name: template sanoid_mail.service
+  become: true
+  ansible.builtin.template:
+    src: "sanoid_mail.service.j2"
+    dest: /etc/systemd/system/sanoid_mail.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - systemctl daemon-reload
+
+- name: add overrides (sanoid_mail + TZ)
+  become: true
+  ansible.builtin.template:
+    src: "overrides.j2"
+    mode: "0644"
+    owner: root
+    group: root
+    dest: /lib/systemd/system/sanoid.service.d/override.conf
+  notify:
+    - systemctl daemon-reload
+
+- name: set timer
+  become: true
+  ansible.builtin.blockinfile:
+    create: true
+    mode: "0644"
+    owner: root
+    group: root
+    path: /lib/systemd/system/sanoid.timer.d/override.conf
+    block: |
+      [Timer]
+      OnCalendar = {{ sanoid_timer }}
+  when: sanoid_timer is defined
+  notify:
+    - systemctl daemon-reload

friedhof/mgrote_zfs_sanoid_mit_syncoid/tasks/source.yml

@@ -0,0 +1,20 @@
+---
+- name: template ssh public key
+  become: true
+  ansible.posix.authorized_key:
+    user: "{{ sanoid_user }}"
+    key: "{{ sanoid_syncoid_ssh_pubkey }}"
+    state: present
+  when:
+    - sanoid_syncoid_source_host
+
+- name: Ensure user is added to sudoers
+  become: true
+  community.general.sudoers:
+    name: "users-sudo-{{ sanoid_user }}"
+    state: present
+    user: "{{ sanoid_user }}"
+    commands: ALL
+    nopassword: true
+  when:
+    - sanoid_syncoid_source_host

friedhof/mgrote_zfs_sanoid_mit_syncoid/tasks/user.yml

@@ -0,0 +1,19 @@
+---
+- name: ensure group exists
+  become: true
+  ansible.builtin.group:
+    name: "{{ sanoid_user_group }}"
+    state: present
+  when:
+    - sanoid_user_group is defined
+    - sanoid_user is defined
+
+- name: ensure user exists
+  become: true
+  ansible.builtin.user:
+    name: "{{ sanoid_user }}"
+    group: "{{ sanoid_user_group }}"
+    create_home: true
+  when:
+    - sanoid_user_group is defined
+    - sanoid_user is defined

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/overrides.j2

@@ -0,0 +1,6 @@
+{{ file_header | default () }}
+
+[Unit]
+OnFailure = sanoid_mail.service
+[Service]
+Environment=TZ=Europe/Berlin

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/private_key.j2

@@ -0,0 +1 @@
+{{ sanoid_syncoid_ssh_privkey }}

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/sanoid.conf.j2

@@ -0,0 +1,25 @@
+{{ file_header | default () }}
+## ZFS Section -------------------------------- ##
+{% for item in sanoid_datasets if item.snapshots is sameas true %}
+[{{ item.path }}]
+  use_template = {{ item.template }}
+  recursive    = {{ item.recursive }}
+## -------------------------------------------- ##
+{% endfor %}
+
+## Template Section --------------------------- ##
+{% for item in sanoid_templates %}
+[template_{{ item.name }}]
+  ## Keep-Rules
+  hourly                = {{ item.keep_hourly }}
+  daily                 = {{ item.keep_daily }}
+  monthly               = {{ item.keep_monthly }}
+  yearly                = {{ item.keep_yearly }}
+  frequently            = {{ item.frequently }}
+  ## Interval
+  frequent_period       = {{ item.frequent_period }}
+  ## Other Options
+  autosnap              = {{ item.autosnap }}
+  autoprune             = {{ item.autoprune }}
+## -------------------------------------------- ##
+{% endfor %}

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/sanoid_mail.service.j2

@@ -0,0 +1,8 @@
+{{ file_header | default () }}
+
+[Unit]
+Description=Send a Mail in case of an error in sanoid.service.
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c '/bin/systemctl status sanoid.service | mail -aFROM:sanoid@mgrote.net -s "[ERROR] sanoid - %H" {{ my_mail }}'

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/syncoid.service.j2

@@ -0,0 +1,10 @@
+{{ file_header | default () }}
+
+[Unit]
+Description=Send zfs snapshots with sanoid/syncoid.
+OnFailure=syncoid_mail.service
+OnSuccess=syncoid_mail.service
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/syncoid.sh

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/syncoid.sh.j2

@@ -0,0 +1,12 @@
+#!/bin/bash
+{{ file_header | default () }}
+
+# check if dest-dataset is mounted (sed: entferne 1. Zeile; awk: zeige nur yes/no; grep: RC1 when != yes)
+{% for item in sanoid_syncoid_datasets_sync %}
+# check if target dataset is mounted
+/usr/sbin/zfs get mounted -H {{ item.destination_mount_check }} 2>&1 > /dev/null || echo "Pool not mounted!"
+# check if source host is reachable
+ping -c1 -W1 {{ item.source_host }} > /dev/null || {{ item.source_host }} not reachable!
+# syncoid
+export HOME=/root ; /usr/bin/syncoid --compress=zstd-fast --sshoption=StrictHostKeyChecking=no --delete-target-snapshots --use-hold --preserve-recordsize --sshkey "/etc/sanoid/.ssh/id_sanoid" --source-bwlimit {{ sanoid_syncoid_bwlimit }} {{ sanoid_user }}@{{ item.source_host }}:{{ item.source_dataset }} {{ item.destination_dataset }}
+{% endfor %}

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/syncoid.timer.j2

@@ -0,0 +1,9 @@
+{{ file_header | default () }}
+[Unit]
+Description=Timer for syncoid.
+
+[Timer]
+OnCalendar={{ sanoid_syncoid_timer }}
+
+[Install]
+WantedBy=timers.target multi-user.target zfs.target

friedhof/mgrote_zfs_sanoid_mit_syncoid/templates/syncoid_mail.service.j2

@@ -0,0 +1,8 @@
+{{ file_header | default () }}
+
+[Unit]
+Description=Send a Mail for sanoid service after error or success sanoid.service.
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c '/usr/bin/journalctl -u syncoid.service -n 30 | mail -aFROM:syncoid@mgrote.net -s "syncoid - %H" {{ my_mail }}'