From a626205272f4a2aa2340b33b2bc631ae26073499 Mon Sep 17 00:00:00 2001 From: Michael Grote Date: Mon, 28 Oct 2024 18:17:23 +0100 Subject: [PATCH] dfgd --- roles/mgrote_rsync/README.md | 85 +++++++++++++++++++ roles/mgrote_rsync/defaults/main.yml | 49 +++++++++++ roles/mgrote_rsync/handlers/main.yml | 5 ++ roles/mgrote_rsync/tasks/destination.yml | 85 +++++++++++++++++++ roles/mgrote_rsync/tasks/main.yml | 78 +++++++++++++++++ roles/mgrote_rsync/tasks/snaps.yml | 46 ++++++++++ roles/mgrote_rsync/tasks/source.yml | 20 +++++ roles/mgrote_rsync/tasks/user.yml | 19 +++++ roles/mgrote_rsync/templates/overrides.j2 | 6 ++ roles/mgrote_rsync/templates/private_key.j2 | 1 + roles/mgrote_rsync/templates/sanoid.conf.j2 | 25 ++++++ .../templates/sanoid_mail.service.j2 | 8 ++ .../mgrote_rsync/templates/syncoid.service.j2 | 10 +++ roles/mgrote_rsync/templates/syncoid.sh.j2 | 12 +++ roles/mgrote_rsync/templates/syncoid.timer.j2 | 9 ++ .../templates/syncoid_mail.service.j2 | 8 ++ 16 files changed, 466 insertions(+) create mode 100644 roles/mgrote_rsync/README.md create mode 100644 roles/mgrote_rsync/defaults/main.yml create mode 100644 roles/mgrote_rsync/handlers/main.yml create mode 100644 roles/mgrote_rsync/tasks/destination.yml create mode 100644 roles/mgrote_rsync/tasks/main.yml create mode 100644 roles/mgrote_rsync/tasks/snaps.yml create mode 100644 roles/mgrote_rsync/tasks/source.yml create mode 100644 roles/mgrote_rsync/tasks/user.yml create mode 100644 roles/mgrote_rsync/templates/overrides.j2 create mode 100644 roles/mgrote_rsync/templates/private_key.j2 create mode 100644 roles/mgrote_rsync/templates/sanoid.conf.j2 create mode 100644 roles/mgrote_rsync/templates/sanoid_mail.service.j2 create mode 100644 roles/mgrote_rsync/templates/syncoid.service.j2 create mode 100644 roles/mgrote_rsync/templates/syncoid.sh.j2 create mode 100644 roles/mgrote_rsync/templates/syncoid.timer.j2 create mode 100644 roles/mgrote_rsync/templates/syncoid_mail.service.j2 diff --git a/roles/mgrote_rsync/README.md b/roles/mgrote_rsync/README.md new file mode 100644 index 00000000..552ed0f0 --- /dev/null +++ b/roles/mgrote_rsync/README.md @@ -0,0 +1,85 @@ +## mgrote.zfs_sanoid + +### Beschreibung +Installiert und konfiguriert ``sanoid`` + ``syncoid``. + +Es gibt 3 Funktionen: + +1. Snapshots erstellen und entfernen +2. Snapshots senden +3. Snapshots empfangen + +### getestet auf +- ProxMox 7.* +- Ubuntu 20.04 + +### Variablen + Defaults +- see [defaults](./defaults/main.yml) + + +### Beispiel Playbook + +```yaml +--- +- hosts: host1,host2 + roles: + - { role: mgrote_zfs_sanoid, tags: "sanoid" } +``` + +### Beispiel - Snapshots erstellen + + +#### Variablen + +```yaml +--- + sanoid_snaps_enable: true + sanoid_datasets: + - path: 'hdd_data/videos' + template: '31tage' + recursive: 'yes' + snapshots: true + sanoid_templates: + - name: '31tage' + keep_hourly: '24' # Aufheben (Stunde) + keep_daily: '31' # Aufheben (Tage) + keep_monthly: '3' # Aufheben (Monate) + keep_yearly: '0' # Aufheben (Jahre) + frequently: '16' # Aufheben (Minuten) + frequent_period: '15' # Intervall (alle 5 Minuten) + autosnap: 'yes' # Automatisches erstellen von Snapshots + autoprune: 'yes' + +``` + +### Beispiel - Snapshots senden und empfangen + +- Host 1 = Source +- Host 2 = Destination + + + +#### Variablen - Host 1 + +```yaml + sanoid_syncoid_source_host: true + sanoid_syncoid_ssh_pubkey: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3U37DGPRPDLlgxZcM0Zj/x6RVZxs7hcWBYfPywujH4+mjbpzJckr2tx3QLfxsCCjQVb4LNSEB0xsOvzDjfDsaPuG4wzqFVyZOtjI4iWg/it4ARndun33r+xSlWc5JKHH9GRK8SBOd4lXv5ylENdhWQ7z5ZF/FtCysb1JHTTYlobgXfTZ4NswJj6BBk669l13uL6zSXq6x6vm1GWiFIcIYqwM5WGSGHFoD2RNn0TJKI9A3AULPloMzWeHG3fJhoVfNY6ZB0kqpTHGoAmJUURkBFki1cJkzx3tyto4VpTzZmUyYg+qqIWbv7Me3YVJCln8JYD10uDb2oPRx6G3C9DlnzRmAVVbqCHzwvOY0H5TLTW7AXCHHgSdHaRym4oTUY9dDS/XFU3rHgexerBbi3sy1Tm0/dEU3cZFm4YOJXY/l4TeTRlhg2VbctsWE1BN1CZcoJRR+qNdJzM7Vl70Y6RGU92Y1rzSpooYVuyCFDrEIp0hAHidb5rs4paCvoxtVqak+LK8dcq0IbWxcxomEimeRG4+Opd3vo+U6subp5jqkOY0uYkFVJXaMHkP5ZIxlCFgif2A3YAPhz9IczRJaaNY3pbVgU7ybOBp+S8KRK8Ysk6OP5ApOTQVTlRhYeNqo7mpuW6139VRY5luekSCy3ehHCI9/MObhu2juF1Nz0HMeMQ== mg@irantu +``` + + +#### Variablen - Host 2 + +```yaml + sanoid_syncoid_timer: '*:*' + sanoid_syncoid_bwlimit: 30m + sanoid_syncoid_datasets_sync: + - source_host: host1.lan + source_dataset: hdd_data_mirror + destination_mount_check: hdd_data/encrypted # Wenn dieses Dataset nicht gemountet ist(z.B. durch Verschlüsselung, dann bricht syncoid ab) + destination_dataset: hdd_data/encrypted/syncoid/zfs1 + skip_parent: false + sanoid_syncoid_ssh_privkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_private_key', 'notes') }}" + sanoid_syncoid_destination_host: true + +``` diff --git a/roles/mgrote_rsync/defaults/main.yml b/roles/mgrote_rsync/defaults/main.yml new file mode 100644 index 00000000..78cf8310 --- /dev/null +++ b/roles/mgrote_rsync/defaults/main.yml @@ -0,0 +1,49 @@ +--- +### when should sanoid be run (every 5 minutes) +sanoid_timer: '*-*-* *:00/5' +### when should syncoid be run +sanoid_syncoid_timer: '*-*-* *:00:00' + +# ### "Default" Datasets +# sanoid_datasets: # dictionary +# - path: 'hdd_data/data' # path to dataset; without leading / +# template: 'fiveminutes' # name +# recursive: 'no' # recursive snapshotting +# snapshots: true # (de)activate; can be used to disable snapshotting of subdatasets if recursive is set +# - path: 'hdd_data/test' +# snapshots: false # deaktiviert sanoid für das dataset +# +# ### Templates +# sanoid_templates: +# - name: 'fiveminutes' +# keep_hourly: '24' # Aufheben (Stunde) +# keep_daily: '31' # Aufheben (Tage) +# keep_monthly: '6' # Aufheben (Monate) +# keep_yearly: '0' # Aufheben (Jahre) +# frequently: '36' # Aufheben (Minuten) +# frequent_period: '5' # Intervall (alle 5 Minuten) +# autosnap: 'yes' # Automatisches erstellen von Snapshots +# autoprune: 'yes' + +### user and group for sanoid +sanoid_user: sanoid +sanoid_user_group: sanoid + +### enable/disable features +## enable snapshotting +# sanoid_snaps_enable: true +## enable sending snaps +# sanoid_syncoid_source_host: true +## enable receiving snaps +# sanoid_syncoid_destination_host: true + +# syncoid +#sanoid_syncoid_ssh_privkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_private_key', 'notes') }}" +#sanoid_syncoid_ssh_pubkey: "{{ lookup('viczem.keepass.keepass', 'sanoid_syncoid_public_key', 'notes') }}" + +### mgrote_sanoid +#sanoid_syncoid_datasets_sync: +# - source_host: pve5.mgrote.net +# source_dataset: hdd_data/tmp +# destination_mount_check: hdd_data/tmp # zielpool +# destination_dataset: backup/pve5/tmp diff --git a/roles/mgrote_rsync/handlers/main.yml b/roles/mgrote_rsync/handlers/main.yml new file mode 100644 index 00000000..ab3f5feb --- /dev/null +++ b/roles/mgrote_rsync/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: systemctl daemon-reload + become: true + ansible.builtin.systemd: + daemon_reload: true diff --git a/roles/mgrote_rsync/tasks/destination.yml b/roles/mgrote_rsync/tasks/destination.yml new file mode 100644 index 00000000..410ae691 --- /dev/null +++ b/roles/mgrote_rsync/tasks/destination.yml @@ -0,0 +1,85 @@ +--- +- name: template ssh private key + become: true + ansible.builtin.template: + src: private_key.j2 + dest: "/etc/sanoid/.ssh/id_sanoid" + owner: "{{ sanoid_user }}" + group: "{{ sanoid_user_group }}" + mode: "0400" + no_log: true + when: + - sanoid_syncoid_destination_host + +- name: Ensure user is added to sudoers + become: true + community.general.sudoers: + name: "users-sudo-{{ sanoid_user }}" + state: present + user: "{{ sanoid_user }}" + commands: ALL + nopassword: true + when: + - sanoid_syncoid_destination_host + +- name: template syncoid.service + become: true + ansible.builtin.template: + src: "syncoid.service.j2" + dest: /etc/systemd/system/syncoid.service + owner: root + group: root + mode: "0644" + notify: + - systemctl daemon-reload + when: + - sanoid_syncoid_destination_host + +- name: template syncoid.sh + become: true + ansible.builtin.template: + src: "syncoid.sh.j2" + dest: /usr/bin/syncoid.sh + owner: root + group: root + mode: "0755" + when: + - sanoid_syncoid_destination_host + +- name: template syncoid_mail.service + become: true + ansible.builtin.template: + src: "syncoid_mail.service.j2" + dest: /etc/systemd/system/syncoid_mail.service + owner: root + group: root + mode: "0644" + notify: + - systemctl daemon-reload + when: + - sanoid_syncoid_destination_host + +- name: template syncoid.timer + become: true + ansible.builtin.template: + src: "syncoid.timer.j2" + dest: "/etc/systemd/system/syncoid.timer" + owner: root + group: root + mode: "0644" + notify: + - systemctl daemon-reload + when: + - sanoid_syncoid_destination_host + +- name: enable syncoid.timer + become: true + ansible.builtin.systemd: + name: "syncoid.timer" + enabled: true + masked: false + state: started + notify: + - systemctl daemon-reload + when: + - sanoid_syncoid_destination_host diff --git a/roles/mgrote_rsync/tasks/main.yml b/roles/mgrote_rsync/tasks/main.yml new file mode 100644 index 00000000..30e0bbd1 --- /dev/null +++ b/roles/mgrote_rsync/tasks/main.yml @@ -0,0 +1,78 @@ +--- +- name: include user tasks + ansible.builtin.include_tasks: user.yml + +- name: install packages from repo + become: true + ansible.builtin.apt: + name: + - mbuffer + - lzop + - libcapture-tiny-perl + - pv + - libconfig-ini-perl + - sanoid + state: present + +- name: Overwrite syncoid script from package + become: true + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/jimsalterjrs/sanoid/master/syncoid + dest: /usr/bin/syncoid + mode: '0755' + owner: root + group: root + force: true + +- name: create sanoid directories + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ sanoid_user }}" + group: "{{ sanoid_user_group }}" + mode: "0700" + with_items: + - "/etc/sanoid" + - "/etc/sanoid/.ssh" + +- name: include snaps tasks + ansible.builtin.include_tasks: snaps.yml + when: + - sanoid_datasets is defined + - sanoid_templates is defined + - sanoid_snaps_enable is defined + - sanoid_snaps_enable + +- name: ensure timers are enabled + become: true + ansible.builtin.systemd: + state: started + name: "{{ item }}" + daemon_reload: true + masked: false + enabled: true + loop: + - sanoid.timer + +- name: ensure services are enabled + become: true + ansible.builtin.systemd: + name: "{{ item }}" + masked: false + enabled: true + loop: + - sanoid.service + - sanoid-prune.service + +- name: include source-host tasks + ansible.builtin.include_tasks: source.yml + when: + - sanoid_syncoid_source_host is defined and sanoid_syncoid_source_host is true + - sanoid_syncoid_ssh_pubkey is defined + +- name: include destination-host tasks + ansible.builtin.include_tasks: destination.yml + when: + - sanoid_syncoid_destination_host is defined and sanoid_syncoid_destination_host is true + - sanoid_syncoid_ssh_privkey is defined diff --git a/roles/mgrote_rsync/tasks/snaps.yml b/roles/mgrote_rsync/tasks/snaps.yml new file mode 100644 index 00000000..5a6b6aac --- /dev/null +++ b/roles/mgrote_rsync/tasks/snaps.yml @@ -0,0 +1,46 @@ +--- +- name: Generate Sanoid Configuration + become: true + ansible.builtin.template: + src: sanoid.conf.j2 + dest: "/etc/sanoid/sanoid.conf" + owner: "{{ sanoid_user }}" + group: "{{ sanoid_user_group }}" + mode: "0400" + +- name: template sanoid_mail.service + become: true + ansible.builtin.template: + src: "sanoid_mail.service.j2" + dest: /etc/systemd/system/sanoid_mail.service + owner: root + group: root + mode: "0644" + notify: + - systemctl daemon-reload + +- name: add overrides (sanoid_mail + TZ) + become: true + ansible.builtin.template: + src: "overrides.j2" + mode: "0644" + owner: root + group: root + dest: /lib/systemd/system/sanoid.service.d/override.conf + notify: + - systemctl daemon-reload + +- name: set timer + become: true + ansible.builtin.blockinfile: + create: true + mode: "0644" + owner: root + group: root + path: /lib/systemd/system/sanoid.timer.d/override.conf + block: | + [Timer] + OnCalendar = {{ sanoid_timer }} + when: sanoid_timer is defined + notify: + - systemctl daemon-reload diff --git a/roles/mgrote_rsync/tasks/source.yml b/roles/mgrote_rsync/tasks/source.yml new file mode 100644 index 00000000..e3d605d2 --- /dev/null +++ b/roles/mgrote_rsync/tasks/source.yml @@ -0,0 +1,20 @@ +--- +- name: template ssh public key + become: true + ansible.posix.authorized_key: + user: "{{ sanoid_user }}" + key: "{{ sanoid_syncoid_ssh_pubkey }}" + state: present + when: + - sanoid_syncoid_source_host + +- name: Ensure user is added to sudoers + become: true + community.general.sudoers: + name: "users-sudo-{{ sanoid_user }}" + state: present + user: "{{ sanoid_user }}" + commands: ALL + nopassword: true + when: + - sanoid_syncoid_source_host diff --git a/roles/mgrote_rsync/tasks/user.yml b/roles/mgrote_rsync/tasks/user.yml new file mode 100644 index 00000000..541c63a2 --- /dev/null +++ b/roles/mgrote_rsync/tasks/user.yml @@ -0,0 +1,19 @@ +--- +- name: ensure group exists + become: true + ansible.builtin.group: + name: "{{ sanoid_user_group }}" + state: present + when: + - sanoid_user_group is defined + - sanoid_user is defined + +- name: ensure user exists + become: true + ansible.builtin.user: + name: "{{ sanoid_user }}" + group: "{{ sanoid_user_group }}" + create_home: true + when: + - sanoid_user_group is defined + - sanoid_user is defined diff --git a/roles/mgrote_rsync/templates/overrides.j2 b/roles/mgrote_rsync/templates/overrides.j2 new file mode 100644 index 00000000..e9033464 --- /dev/null +++ b/roles/mgrote_rsync/templates/overrides.j2 @@ -0,0 +1,6 @@ +{{ file_header | default () }} + +[Unit] +OnFailure = sanoid_mail.service +[Service] +Environment=TZ=Europe/Berlin diff --git a/roles/mgrote_rsync/templates/private_key.j2 b/roles/mgrote_rsync/templates/private_key.j2 new file mode 100644 index 00000000..de719dbf --- /dev/null +++ b/roles/mgrote_rsync/templates/private_key.j2 @@ -0,0 +1 @@ +{{ sanoid_syncoid_ssh_privkey }} diff --git a/roles/mgrote_rsync/templates/sanoid.conf.j2 b/roles/mgrote_rsync/templates/sanoid.conf.j2 new file mode 100644 index 00000000..26fa7f80 --- /dev/null +++ b/roles/mgrote_rsync/templates/sanoid.conf.j2 @@ -0,0 +1,25 @@ +{{ file_header | default () }} +## ZFS Section -------------------------------- ## +{% for item in sanoid_datasets if item.snapshots is sameas true %} +[{{ item.path }}] + use_template = {{ item.template }} + recursive = {{ item.recursive }} +## -------------------------------------------- ## +{% endfor %} + +## Template Section --------------------------- ## +{% for item in sanoid_templates %} +[template_{{ item.name }}] + ## Keep-Rules + hourly = {{ item.keep_hourly }} + daily = {{ item.keep_daily }} + monthly = {{ item.keep_monthly }} + yearly = {{ item.keep_yearly }} + frequently = {{ item.frequently }} + ## Interval + frequent_period = {{ item.frequent_period }} + ## Other Options + autosnap = {{ item.autosnap }} + autoprune = {{ item.autoprune }} +## -------------------------------------------- ## +{% endfor %} diff --git a/roles/mgrote_rsync/templates/sanoid_mail.service.j2 b/roles/mgrote_rsync/templates/sanoid_mail.service.j2 new file mode 100644 index 00000000..3c3a9727 --- /dev/null +++ b/roles/mgrote_rsync/templates/sanoid_mail.service.j2 @@ -0,0 +1,8 @@ +{{ file_header | default () }} + +[Unit] +Description=Send a Mail in case of an error in sanoid.service. + +[Service] +Type=oneshot +ExecStart=/bin/bash -c '/bin/systemctl status sanoid.service | mail -aFROM:sanoid@mgrote.net -s "[ERROR] sanoid - %H" {{ my_mail }}' diff --git a/roles/mgrote_rsync/templates/syncoid.service.j2 b/roles/mgrote_rsync/templates/syncoid.service.j2 new file mode 100644 index 00000000..af61c7f5 --- /dev/null +++ b/roles/mgrote_rsync/templates/syncoid.service.j2 @@ -0,0 +1,10 @@ +{{ file_header | default () }} + +[Unit] +Description=Send zfs snapshots with sanoid/syncoid. +OnFailure=syncoid_mail.service +OnSuccess=syncoid_mail.service + +[Service] +Type=simple +ExecStart=/usr/bin/syncoid.sh diff --git a/roles/mgrote_rsync/templates/syncoid.sh.j2 b/roles/mgrote_rsync/templates/syncoid.sh.j2 new file mode 100644 index 00000000..68b2f752 --- /dev/null +++ b/roles/mgrote_rsync/templates/syncoid.sh.j2 @@ -0,0 +1,12 @@ +#!/bin/bash +{{ file_header | default () }} + +# check if dest-dataset is mounted (sed: entferne 1. Zeile; awk: zeige nur yes/no; grep: RC1 when != yes) +{% for item in sanoid_syncoid_datasets_sync %} +# check if target dataset is mounted +/usr/sbin/zfs get mounted -H {{ item.destination_mount_check }} 2>&1 > /dev/null || echo "Pool not mounted!" +# check if source host is reachable +ping -c1 -W1 {{ item.source_host }} > /dev/null || {{ item.source_host }} not reachable! +# syncoid +export HOME=/root ; /usr/bin/syncoid --compress=zstd-fast --sshoption=StrictHostKeyChecking=no --delete-target-snapshots --use-hold --preserve-recordsize --sshkey "/etc/sanoid/.ssh/id_sanoid" --source-bwlimit {{ sanoid_syncoid_bwlimit }} {{ sanoid_user }}@{{ item.source_host }}:{{ item.source_dataset }} {{ item.destination_dataset }} +{% endfor %} diff --git a/roles/mgrote_rsync/templates/syncoid.timer.j2 b/roles/mgrote_rsync/templates/syncoid.timer.j2 new file mode 100644 index 00000000..16438ec7 --- /dev/null +++ b/roles/mgrote_rsync/templates/syncoid.timer.j2 @@ -0,0 +1,9 @@ +{{ file_header | default () }} +[Unit] +Description=Timer for syncoid. + +[Timer] +OnCalendar={{ sanoid_syncoid_timer }} + +[Install] +WantedBy=timers.target multi-user.target zfs.target diff --git a/roles/mgrote_rsync/templates/syncoid_mail.service.j2 b/roles/mgrote_rsync/templates/syncoid_mail.service.j2 new file mode 100644 index 00000000..caa59b24 --- /dev/null +++ b/roles/mgrote_rsync/templates/syncoid_mail.service.j2 @@ -0,0 +1,8 @@ +{{ file_header | default () }} + +[Unit] +Description=Send a Mail for sanoid service after error or success sanoid.service. + +[Service] +Type=oneshot +ExecStart=/bin/bash -c '/usr/bin/journalctl -u syncoid.service -n 30 | mail -aFROM:syncoid@mgrote.net -s "syncoid - %H" {{ my_mail }}'