setup minio automation (#229)

Reviewed-on: #229
Co-authored-by: Michael Grote <michael.grote@posteo.de>
Co-committed-by: Michael Grote <michael.grote@posteo.de>

.ansible-lint

@@ -15,3 +15,7 @@ exclude_paths:
   - playbooks/3_service
   - playbooks/1_bootstrap.yml
   - .gitea/
+mock_modules:
+  - dubzland.minio.minio_alias
+  - dubzland.minio.minio_user
+  - dubzland.minio.minio_bucket

group_vars/docker.yml

@@ -1,4 +1,17 @@
 ---
+### mgrote_minio_configure
+minio_url: https://s3.mgrote.net
+minio_root_access_key: "{{ lookup('viczem.keepass.keepass', 'minio_root_access_key', 'password') }}"
+minio_root_secret_key: "{{ lookup('viczem.keepass.keepass', 'minio_root_secret_key', 'password') }}"
+minio_users:
+  - name: testuser
+    secret: "{{ lookup('viczem.keepass.keepass', 'minio_testuser_secret_key', 'password') }}"
+    state: present
+    policy: testbucket_rw
+minio_buckets:
+  - name: testbucket
+    state: present
+
 ### mrlesmithjr.manage_lvm
 lvm_groups:
   - vgname: vg_docker
@@ -19,6 +32,7 @@ pvresize_to_max: true
 pip_package: python3-pip
 pip_install_packages:
   - name: docker # für munin-plugin docker_
+  - name: minio # für ansible-minio_configure-Rolle
 
 ### mgrote.apt_manage_packages
 apt_packages_extra:

playbooks/3_service/docker.yml

@@ -25,3 +25,6 @@
     - role: mgrote_docker_housekeeping
       tags: "housekeeping"
       become: true
+    - role: mgrote_minio_configure
+      tags: "minio"
+      become: true

roles/mgrote_minio_configure/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+# Client
+minio_client_release: ""
+minio_client_bin: /usr/local/bin/mc
+
+# Allgemeines
+minio_config_dir: /etc/minio
+minio_root_alias: minio_ansible_root
+
+minio_url: https://s3.mgrote.net
+
+# Secrets
+minio_root_access_key: pfiDnXXXXXedZbD
+minio_root_secret_key: Bc2exxxxxxxxxxxxxYRAfV
+
+# User
+minio_users:
+  - name: testuser5
+    secret: hallowelt
+    state: present
+    policy: testbucket1_rw
+  - name: testuser6
+    secret: hallowelt2
+    state: present
+    policy: testbucket3_rw
+  - name: testuserwer
+    secret: hallowelt2
+    state: absent
+    policy: testbucket3_ro
+
+# Buckets
+# Anzulegende Buckets, pro Bucket wird eine RW + RO Policy erstellt
+minio_buckets:
+  - name: testbucket1
+    state: present
+  - name: testbucket3
+    state: absent

roles/mgrote_minio_configure/tasks/bucket.yml

@@ -0,0 +1,16 @@
+---
+- name: Setup buckets
+  dubzland.minio.minio_bucket:
+    name: "{{ item.name }}"
+    auth:
+      access_key: "{{ minio_root_access_key }}"
+      secret_key: "{{ minio_root_secret_key }}"
+      url: "{{ minio_url }}"
+    state: "{{ item.state | default('present') }}"
+  loop: "{{ minio_buckets }}"
+
+- name: Deletion Info
+  ansible.builtin.debug:
+    msg: "Deletion only possible with empty bucket ({{ item.name }})"
+  when: '"absent" in item.state'
+  loop: "{{ minio_buckets }}"

roles/mgrote_minio_configure/tasks/client.yml

@@ -0,0 +1,32 @@
+---
+# stolen and adapted from: https://github.com/atosatto/ansible-minio/blob/master/tasks/install-client.yml
+- name: Compose the Minio client download base url
+  ansible.builtin.set_fact:
+    _minio_client_download_base_url: "https://dl.minio.io/client/mc/release/linux-amd64"
+
+- name: Compose the Minio client download url with lastest release
+  ansible.builtin.set_fact:
+    _minio_client_download_url: "{{ _minio_client_download_base_url }}/mc"
+  when: minio_client_release | length == 0
+
+- name: "Compose the Minio client download url with release {{ minio_client_release }}"
+  ansible.builtin.set_fact:
+    _minio_client_download_url: "{{ _minio_client_download_base_url }}/archive/mc.{{ minio_client_release }}"
+  when: minio_client_release | length > 0
+
+- name: "Get the Minio client checksum for amd64 architecture"
+  ansible.builtin.set_fact:
+    _minio_client_checksum: "{{ lookup('url', _minio_client_download_url + '.sha256sum').split(' ')[0] }}"
+
+- name: Download the Minio client
+  ansible.builtin.get_url:
+    url: "{{ _minio_client_download_url }}"
+    dest: "{{ minio_client_bin }}"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+    checksum: "sha256:{{ _minio_client_checksum }}"
+  register: _download_client
+  until: _download_client is succeeded
+  retries: 5
+  delay: 2

roles/mgrote_minio_configure/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+# auf Zielhost benötigt: pip3 install minio
+# auf ansible-controller: https://galaxy.ansible.com/ui/repo/published/dubzland/minio/content/module/minio_alias/
+
+- name: include mcli tasks
+  ansible.builtin.include_tasks: client.yml
+
+- name: ensure mc alias exists
+  dubzland.minio.minio_alias:
+    name: "{{ minio_root_alias }}"
+    url: "{{ minio_url }}"
+    access_key: "{{ minio_root_access_key }}"
+    secret_key: "{{ minio_root_secret_key }}"
+    state: present
+
+- name: include policy tasks
+  ansible.builtin.include_tasks: policy.yml
+
+- name: include bucket tasks
+  ansible.builtin.include_tasks: bucket.yml
+
+- name: Ensure users exist
+  dubzland.minio.minio_user:
+    access_key: "{{ item.name }}"
+    secret_key: "{{ item.secret }}"
+    policy: "{{ item.policy }}"
+    auth:
+      access_key: "{{ minio_root_access_key }}"
+      secret_key: "{{ minio_root_secret_key }}"
+      url: "{{ minio_url }}"
+    state: "{{ item.state | default('present') }}"
+  loop: "{{ minio_users }}"
+  no_log: true
+
+# Bug: delegate_to: localhost
+# in ansible-devspace wird das Python Paket "minio" nicht gefunden

roles/mgrote_minio_configure/tasks/policy.yml

@@ -0,0 +1,44 @@
+---
+# https://galaxy.ansible.com/ui/repo/published/dubzland/minio/content/module/minio_policy/ ?
+- name: "ensure needed dirs exist"
+  ansible.builtin.file:
+    path: "{{ minio_config_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: "prep: template policy files (ro)"
+  ansible.builtin.template:
+    dest: "{{ minio_config_dir }}/{{ item.name }}_ro"
+    src: policy_ro.j2
+    owner: root
+    group: root
+    mode: '0644'
+  loop: "{{ minio_buckets }}"
+
+- name: "prep: template policy files (rw)"
+  ansible.builtin.template:
+    dest: "{{ minio_config_dir }}/{{ item.name }}_rw"
+    src: policy_rw.j2
+    owner: root
+    group: root
+    mode: '0644'
+  loop: "{{ minio_buckets }}"
+
+- name: "setup policies (ro)"
+  ansible.builtin.command: "{{ minio_client_bin }} --disable-pager admin policy create {{ minio_root_alias }} {{ item.name }}_ro {{ minio_config_dir }}/{{ item.name }}_ro"
+  loop: "{{ minio_buckets }}"
+  changed_when: false # Befehl gibt immer "Created policy `testbucket3_ro` successfully." aus, unabhängig ob sie schon existiert oder nicht.
+
+- name: "setup policies (rw)"
+  ansible.builtin.command: "{{ minio_client_bin }} --disable-pager admin policy create {{ minio_root_alias }} {{ item.name }}_rw {{ minio_config_dir }}/{{ item.name }}_rw"
+  loop: "{{ minio_buckets }}"
+  changed_when: false # Befehl gibt immer "Created policy `testbucket3_ro` successfully." aus, unabhängig ob sie schon existiert oder nicht.
+
+- name: "remove old policy files"
+  ansible.builtin.file:
+    path: "{{ minio_config_dir }}/{{ item.name }}*"
+    state: absent
+  loop: "{{ minio_buckets }}"
+  when: '"absent" in item.state'

roles/mgrote_minio_configure/templates/policy_ro.j2

@@ -0,0 +1,37 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads"
+      ],
+      "Resource": [
+        "arn:aws:s3:::{{ item.name }}"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetObject",
+        "s3:ListMultipartUploadParts"
+      ],
+      "Resource": [
+        "arn:aws:s3:::{{ item.name }}/*"
+      ]
+    }
+  ]
+}

roles/mgrote_minio_configure/templates/policy_rw.j2

@@ -0,0 +1,39 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads"
+      ],
+      "Resource": [
+        "arn:aws:s3:::{{ item.name }}"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:DeleteObject",
+        "s3:GetObject",
+        "s3:ListMultipartUploadParts",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::{{ item.name }}/*"
+      ]
+    }
+  ]
+}