bdfr + bugfixes (#115)

lock für script

log vars

doku

changed never

pool test

bdfr log

bdfr log

reboot cifs

mount

vars

aufräumen

download script

tasks

playbook

group vars

doku

eiegenes dataset archiv

keepass bfdr invalid

keepass: bfdr client id + client secret

upvoted saved and liked

Leerzeichen

sources bug

inventory

vars

vm test cv5pve

typo

Linux MInt fur Apt-Manage Sources

bdfr share

KeePassPW angepasst bdfr

playbook

bug cifs

bdfr

nutzer fileserver

Keepass: user bdfr

Co-authored-by: Michael Grote <michael.grote@posteo.de>
Reviewed-on: mg/ansible#115
Co-Authored-By: mg <mg@noreply.git.mgrote.net>
Co-Committed-By: mg <mg@noreply.git.mgrote.net>

group_vars/bdfr.yml

@@ -0,0 +1,17 @@
+---
+  # mgrote.mount_cifs
+  cifs_mounts:
+    - name: fileserver2_bdfr_mount
+      type: cifs
+      state: present
+      dest: "{{ bdfr_data_path }}"
+      src: //fileserver2.grote.lan/archiv/reddit
+      user: bdfr
+      password: "{{ lookup('keepass', 'fileserver_smb_user_bdfr', 'password') }}"
+      domain: grote.lan
+      uid: 1000
+      gid: 1000
+  # mgrote.bdfr
+  bdfr_data_path: /mnt/bdfr
+  bdfr_data_mount: /mnt/bdfr
+  bdfr_log_file: /var/log/bdfr.log

group_vars/fileserver.yml

@@ -6,6 +6,7 @@
   # pct set 109 -mp3 /hdd_data_raidz/papa_backup,mp=/shares_papa_backup
   # pct set 109 -mp4 /hdd_data_raidz/music,mp=/shares_music
   # pct set 109 -mp5 /hdd_data_raidz/tmp,mp=/shares_tmp
+  # pct set 109 -mp6 /hdd_data_raidz/archiv,mp=/shares_archiv
   # Bind Mounts - fileserver-test
   # pct set 158 -mp0 /rpool/vm/dir/vm-158/videos,mp=/shares_videos
   # pct set 158 -mp1 /rpool/vm/dir/vm-158/data,mp=/shares
@@ -13,6 +14,8 @@
   # pct set 158 -mp3 /rpool/vm/dir/vm-158/papa,mp=/shares_papa_backup
   # pct set 158 -mp4 /rpool/vm/dir/vm-158/music,mp=/shares_music
   # pct set 158 -mp5 /rpool/vm/dir/vm-158/tmp,mp=/shares_tmp
+  # pct set 158 -mp6 /rpool/vm/dir/vm-158/archiv,mp=/shares_archiv
+
   ### mgrote.smb_fileserver
   smb_users:
     - name: 'annemariedroessler'
@@ -35,6 +38,9 @@
       password: "{{ lookup('keepass', 'fileserver_smb_user_brother_ads2700w', 'password') }}"
     - name: 'photoprism'
       password: "{{ lookup('keepass', 'fileserver_smb_user_photoprism', 'password') }}"
+    - name: 'bdfr'
+      password: "{{ lookup('keepass', 'fileserver_smb_user_bdfr', 'password') }}"
+
   smb_shares:
     - name: 'videos'
       path: '/shares_videos'
@@ -56,6 +62,10 @@
       path: '/shares/Backup'
       users_ro: ''
       users_rw: 'annemariedroessler restic win10 michaelgrote'
+    - name: 'archiv'
+      path: '/shares_archiv'
+      users_ro: ''
+      users_rw: 'bdfr michaelgrote'
     - name: 'hm'
       path: '/shares/hm'
       users_ro: 'win10'

host_vars/pve2.grote.lan.yml

@@ -88,6 +88,14 @@
       dnodesize: auto
       atime: on
       snapdir: hidden
+    - dataset: hdd_data_raidz/archiv
+      state: present
+      compression: zstd
+      sync: disabled
+      xattr: sa
+      dnodesize: auto
+      atime: on
+      snapdir: hidden
     - dataset: rpool/vm/dir/vm-158/data
       state: present
       compression: lz4
@@ -136,6 +144,14 @@
       dnodesize: auto
       atime: on
       snapdir: hidden
+    - dataset: rpool/vm/dir/vm-158/archiv
+      state: present
+      compression: lz4
+      sync: disabled
+      xattr: sa
+      dnodesize: auto
+      atime: on
+      snapdir: hidden
 
   ### mgrote.apcupsd
   apcupsd_nis_master: true
@@ -182,6 +198,10 @@
       snapshots: true
     - path: 'hdd_data_raidz/pve_backup'
       snapshots: false # deaktiviert sanoid für das dataset
+    - path: 'hdd_data_raidz/archiv'
+      template: '3monate'
+      recursive: 'yes'
+      snapshots: true
   sanoid_templates:
     - name: '31tage'
       keep_hourly: '24'                        # Aufheben (Stunde)
@@ -210,9 +230,18 @@
       frequent_period: '0'
       autosnap: 'yes'
       autoprune: 'yes'
+    - name: '3monate'
+      keep_hourly: '0'
+      keep_daily: '3'
+      keep_monthly: '3'
+      keep_yearly: '0'
+      frequently: '0'
+      frequent_period: '0'
+      autosnap: 'yes'
+      autoprune: 'yes'
   ### mgrote.cv4pve-autosnap
   cv4pve_api_user: root@pam!cv4pve-autosnap
   cv4pve_api_token: "{{ lookup('keepass', 'cv4pve_api_token', 'password') }}"
-  cv4pve_vmid: all,-109,-158,-131
+  cv4pve_vmid: all,-109,-158,-112
   cv4pve_keep_snapshots: 1
   cv4pve_dl_link: "https://github.com/Corsinvest/cv4pve-autosnap/releases/download/v1.10.0/cv4pve-autosnap-linux-x64.zip"

inventory

@@ -28,6 +28,9 @@ all:
       hosts:
         docker.grote.lan:
         docker-test.grote.lan:
+    bdfr:
+      hosts:
+        bdfr.grote.lan:
     proxmox:
       hosts:
         pve2-test.grote.lan:
@@ -70,6 +73,7 @@ all:
         gitea.grote.lan:
         pihole2.grote.lan:
         ntp-server.grote.lan:
+        bdfr.grote.lan:
     test:
       hosts:
         dokuwiki-test.grote.lan:

playbooks/3_service/bdfr.yml

@@ -0,0 +1,5 @@
+---
+- hosts: bdfr
+  roles:
+      - { role: mgrote.mount_cifs, tags: "cifs" }
+      - { role: mgrote.bdfr, tags: "cifs" }

roles/mgrote.apt_manage_sources/tasks/main.yml

@@ -3,6 +3,10 @@
     include_tasks: ubuntu.yml
     when: ansible_distribution == 'Ubuntu'
 
+  - name: include Linux Mint tasks (determined by "ansible_distribution")
+    include_tasks: mint.yml
+    when: ansible_distribution == 'Linux Mint'
+
   - name: include proxmox tasks (determined by group)
     include_tasks: proxmox.yml
     when: "'proxmox' in group_names"

roles/mgrote.apt_manage_sources/tasks/mint.yml

@@ -0,0 +1,8 @@
+---
+  - name: Mint - copy sources.list
+    become: yes
+    ansible.builtin.template:
+      src: "mint_sources.list"
+      dest: "/etc/apt/sources.list"
+      backup: yes
+    register: copy_src

roles/mgrote.apt_manage_sources/templates/mint_sources.list

@@ -0,0 +1,9 @@
+{{ file_header | default () }}
+deb http://{{ manage_sources_apt_proxy_url }}packages.linuxmint.com ulyssa main upstream import backport
+
+deb http://{{ manage_sources_apt_proxy_url }}archive.ubuntu.com/ubuntu focal main restricted universe multiverse
+deb http://{{ manage_sources_apt_proxy_url }}archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse
+deb http://{{ manage_sources_apt_proxy_url }}archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse
+
+deb http://{{ manage_sources_apt_proxy_url }}security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse
+deb http://{{ manage_sources_apt_proxy_url }}archive.canonical.com/ubuntu/ focal partner

roles/mgrote.bdfr/README.md

@@ -0,0 +1,23 @@
+## mgrote.bdfr
+
+Installiert [Bulk Downloader for Reddit](https://github.com/aliparlakci/bulk-downloader-for-reddit#configuration).
+Erstellt einen Cronjob.
+
+__Benötigt einen Webbrowser!__
+
+### Funktioniert auf
+- [ Linux Mint (>20.1)
+
+
+### Variablen + Defaults
+see [defaults](./defaults/main.yml)
+
+
+## Script einrichten(Token von Reddit holen)
+1. Script ausführen `bdfr_download.sh`
+2. Link anklicken
+3. bei Reddit anmelden
+4. Download...
+
+## to HTML
+* https://github.com/BlipRanger/bdfr-html

roles/mgrote.bdfr/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+  bdfr_data_path: /mnt/bdfr/reddit
+  bdfr_data_mount: /mnt/bdfr
+  bdfr_log_file: /var/log/bdfr.log

roles/mgrote.bdfr/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+  - name: install python3.9
+    become: yes
+    ansible.builtin.package:
+      name:
+        - python3.9
+        - python3-pip
+      state: present
+
+  - name: Install bdfr
+    become: yes
+    shell: python3.9 -m pip install bdfr --upgrade
+    changed_when: False
+
+  - name: copy bdfr_download.sh
+    become: yes
+    ansible.builtin.template:
+      src: "bdfr_download.sh"
+      dest: "/usr/local/bin/bdfr_download.sh"
+      mode: 0755
+      owner: mg
+      group: mg
+
+  - name: create cronjob
+    become: true
+    ansible.builtin.cron:
+      name: bdfr
+      minute: "41"
+      hour: "2"
+      job: "/usr/local/bin/bdfr_download.sh >> {{ bdfr_log_file }}"
+      state: present
+
+  - name: copy logrotate config
+    become: yes
+    ansible.builtin.template:
+      src: logrotate_config
+      dest: /etc/logrotate.d/bdfr
+      owner: root
+      group: root
+      mode: 0644

roles/mgrote.bdfr/templates/bdfr_download.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+{{ file_header | default () }}
+
+chmod +x /home/mg/.config/bdfr/default_config.cfg
+
+# These functions return exit codes: 0 = found, 1 = not found
+# https://serverfault.com/questions/50585/whats-the-best-way-to-check-if-a-volume-is-mounted-in-a-bash-script
+isMounted    () { findmnt -rno SOURCE,TARGET "$1" >/dev/null;} #path or device
+
+#!/bin/bash
+{{ file_header | default () }}
+LOCKDIR=${HOME}/.cache # set lockdir
+function exlock() { # define Function for setting lock; stops the script i a lock exists
+    exec {lock_fd}>${LOCKDIR}/$(basename $0).lock
+    flock -nx "$lock_fd"
+    if [[ $? == 1 ]]; then
+        exit 1
+    fi
+}
+function unlock() { # define function for removing lock
+    rm "${LOCKDIR}/$(basename $0).lock"
+    [[ -n $1 ]] && exit $1
+    exit
+}
+
+exlock # set lock
+if isMounted "{{ bdfr_data_mount }}";
+   then
+      python3.9 -m bdfr clone {{ bdfr_data_path }} --no-dupes --search-existing --submitted --upvoted --saved --authenticate --user me
+      python3.9 -m bdfr clone {{ bdfr_data_path }} --no-dupes --search-existing --subreddit SCPDeclassified
+   else echo "share is not mounted"
+      exit 1
+fi
+
+unlock # entferne lock

roles/mgrote.bdfr/templates/logrotate_config

@@ -0,0 +1,13 @@
+{{ bdfr_log_file }} {
+    missingok
+    nocreate
+    rotate 4
+    weekly
+    compress
+    missingok
+    notifempty
+    dateext
+    dateyesterday
+    sharedscripts
+    endscript
+}

roles/mgrote.mount_cifs/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+  - name: reboot
+    become: true
+    reboot:
+      reboot_timeout: 120

roles/mgrote.mount_cifs/tasks/main.yml

@@ -12,6 +12,7 @@
       state: directory
       mode: 0777
     loop: "{{ cifs_mounts }}"
+    no_log: true
 
   - name: create credential-file
     become: yes
@@ -29,7 +30,7 @@
     ansible.posix.mount:
       src: "{{ item.src }}"
       path: "{{ item.dest }}"
-      opts: credentials=/root/.smb_passwords_{{ item.name }},domain={{ item.domain }},uid={{ item.uid | default('1000') }}",gid={{ item.gid | default('1000') }}"
+      opts: credentials=/root/.smb_passwords_{{ item.name }},domain={{ item.domain }},uid={{ item.uid | default('1000') }},gid={{ item.gid | default('1000') }}"
       state: "{{ item.state }}"
       fstype: "{{ item.type }}"
       backup: yes
@@ -37,3 +38,4 @@
       passno: "0"
     loop: "{{ cifs_mounts }}"
     no_log: true
+    notify: reboot

roles/mgrote.r8152_kernel_module/tasks/main.yml

@@ -11,6 +11,6 @@
     register: module_in_lsmod
     changed_when: False # sorgt dafür das der Task nie als "changed" angezeigt wird
 
-  - name: include tasks (taks get skipped if the folder or the module exists or r8152_module_needed == false)
+  - name: include tasks (task get skipped if the folder or the module exists or r8152_module_needed == false)
     include_tasks: doing.yml
     when: ((folder_exists.stat.exists == False) or (module_in_lsmod.stdout.find('r8152') == -1)) and (r8152_module_needed == true)

roles/mgrote.vim/tasks/main.yml

@@ -13,7 +13,7 @@
       path: "{{ vim_vundle_path }}"
     register: folder_exists
 
-  - name: Ensure  repository is cloned locally.
+  - name: Ensure repository is cloned locally.
     git:
       repo: "{{ vim_vundle_repo_url }}"
       dest: "{{ vim_vundle_path }}"