Backup using Talos + Velero

applications.yml

@@ -8,3 +8,6 @@
       tags:
         - haproxy
         - haproxy_control_lb
+    - role: backup_talos
+      tags:
+        - backup_talos

roles/backup_talos/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+backup_user: talosbak
+backup_home: "/home/{{ backup_user }}"

roles/backup_talos/files/talosbackup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+IFS=$'\n'
+set -o xtrace -o errexit -o errtrace -o nounset -o pipefail +o history
+umask 0077
+
+WORK_DIR="${HOME}"
+SUB_DIR="talos"
+SNAPSHOT_ARCHIVE="talos_day$(date +%d).tgz"
+
+mkdir -p "${WORK_DIR}/${SUB_DIR}"
+cd "${WORK_DIR}/${SUB_DIR}"
+talosctl etcd snapshot "db.snapshot" &>"db.snapshot.log"
+talosctl get members --output yaml >"members.yaml"
+while read -r NODE_IP; do
+  talosctl get machineconfig -o yaml -n "${NODE_IP}" >"machineconfig.${NODE_IP}.yaml"
+done < <(talosctl get members --output json | jq -r '.spec.addresses[] | select(test("^[0-9\\.]+$"))')
+
+cd "${WORK_DIR}"
+tar -cpzf "${SNAPSHOT_ARCHIVE}" "${SUB_DIR}"
+rm  -rf  "${WORK_DIR}/${SUB_DIR}"
+
+ls -lah "${SNAPSHOT_ARCHIVE}"
+
+echo "Backup completed - $(date)"

roles/backup_talos/files/talosbackup.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Talos Backup Timer
+After=multi-user.target
+
+[Timer]
+OnCalendar=01:10
+
+[Install]
+WantedBy=timers.target

roles/backup_talos/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+- name: Install requirements
+  ansible.builtin.apt:
+    name:
+      - jq
+      - tar
+      - gzip
+    install_recommends: false
+
+- name: Install talosctl
+  ansible.builtin.get_url:
+    url: "https://github.com/siderolabs/talos/releases/download/{{ talos_version }}/talosctl-{{ ansible_system | lower }}-{{ deb_architecture[ansible_architecture] }}"
+    dest: /usr/local/bin/talosctl
+    mode: u=rwx,go=rx
+
+- name: Add the talosbak user
+  ansible.builtin.user:
+    name: "{{ backup_user }}"
+    comment: "Talos Backup User"
+    system: true
+    home: "{{ backup_home }}"
+
+- name: Directory /home/talosbak/.talos
+  ansible.builtin.file:
+    state: directory
+    path: "{{ backup_home }}/.talos"
+    owner: "{{ backup_user }}"
+    mode: u=rwx,go=
+
+- name: Copy talos config
+  ansible.builtin.copy:
+    src: "{{ talosconfig }}"
+    dest: "{{ backup_home }}/.talos/config"
+    owner: "{{ backup_user }}"
+    mode: u=rx,go=
+
+- name: Directory /home/talosbak/bin
+  ansible.builtin.file:
+    state: directory
+    path: "{{ backup_home }}/bin"
+    owner: "{{ backup_user }}"
+    mode: u=rx,go=
+
+- name: Copy talosbackup.sh
+  ansible.builtin.copy:
+    src: "talosbackup.sh"
+    dest: "{{ backup_home }}/bin/talosbackup.sh"
+    owner: "{{ backup_user }}"
+    mode: u=rx,go=
+
+- name: Copy talosbackup.timer
+  ansible.builtin.copy:
+    src: "talosbackup.timer"
+    dest: "/etc/systemd/system/talosbackup.timer"
+    mode: u=rw,go=r
+
+- name: Copy talosbackup.service
+  ansible.builtin.template:
+    src: "talosbackup.service.j2"
+    dest: "/etc/systemd/system/talosbackup.service"
+    mode: u=rw,go=r
+
+- name: Enable talosbackup.timer
+  ansible.builtin.systemd:
+    name: talosbackup.timer
+    enabled: true
+    state: started
+    daemon_reload: true

roles/backup_talos/templates/talosbackup.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Talos Backup
+After=network-online.service multi-user.target
+
+[Service]
+Type=oneshot
+User={{ backup_user }}
+Group={{ backup_user }}
+ExecStart={{ backup_home }}/bin/talosbackup.sh
+SyslogIdentifier=%N
+IOWeight=1
+Nice=19

roles/haproxy_control_lb/tasks/firewall.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Get IP of Ansible controller
   ansible.builtin.set_fact:
     _ssh_client_ip: "{{ ansible_env.SSH_CLIENT.split(' ')[0] }}"
@@ -9,14 +8,23 @@
     rule: allow
     port: "50000"
     src: "{{ item }}"
-  loop: "{{ ip_whitelist + [_ssh_client_ip] }}"
+  loop: "{{ ip_whitelist + [_ssh_client_ip, '127.0.0.1', '::1'] }}"
+
+- name: Allow access from nodes to tcp port 50000
+  community.general.ufw:
+    rule: allow
+    port: "50000"
+    src: "{{ hostvars[talos_node].ansible_host }}"
+  loop: "{{ groups['customer_' + cluster_name_short] }}"
+  loop_control:
+    loop_var: talos_node
 
 - name: Allow access from nodes to tcp port 50001
   community.general.ufw:
     rule: allow
     port: "50001"
     src: "{{ hostvars[talos_node].ansible_host }}"
-  loop: "{{ groups['customer_' + customer] }}"
+  loop: "{{ groups['customer_' + cluster_name_short] }}"
   loop_control:
     loop_var: talos_node
 

roles/haproxy_control_lb/templates/haproxy_control_lb.cfg.j2

@@ -7,9 +7,9 @@ frontend tcp_talosapi
 backend bk_talosapi
     mode tcp
     balance leastconn
-{% for talos_node in groups['customer_'+customer] %}
+{% for talos_node in groups['customer_' + cluster_name_short] %}
 {%   if talos_node in groups['talos_control_nodes'] %}
-    server  {{ talos_node }}  {{ hostvars[talos_node].ansible_host }}:50000  check
+    server  {{ talos_node }}  {{ hostvars[talos_node].ansible_host }}:50000
 {%   endif %}
 {% endfor %}
 
@@ -24,9 +24,9 @@ frontend tcp_talostrustd
 backend bk_talostrustd
     mode tcp
     balance leastconn
-{% for talos_node in groups['customer_'+customer] %}
+{% for talos_node in groups['customer_' + cluster_name_short] %}
 {%   if talos_node in groups['talos_control_nodes'] %}
-    server  {{ talos_node }}  {{ hostvars[talos_node].ansible_host }}:50001  check
+    server  {{ talos_node }}  {{ hostvars[talos_node].ansible_host }}:50001
 {%   endif %}
 {% endfor %}
 
@@ -41,7 +41,7 @@ frontend tcp_kubeapi
 backend bk_kubeapi
     mode tcp
     balance leastconn
-{% for talos_node in groups['customer_'+customer] %}
+{% for talos_node in groups['customer_' + cluster_name_short] %}
 {%   if talos_node in groups['talos_control_nodes'] %}
     server  {{ talos_node }}  {{ hostvars[talos_node].ansible_host }}:6443  check
 {%   endif %}

roles/velero/tasks/main.yml

@@ -31,6 +31,7 @@
     wait: true
     # https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml
     values:
+      snapshotsEnabled: false
       initContainers:
         - name: velero-plugin-for-aws
           image: velero/velero-plugin-for-aws:latest
@@ -51,27 +52,14 @@
           - name: wasabi
             provider: velero.io/aws
             bucket: snelcom-velero
+            prefix: "{{ cluster_name_short }}"
             default: true
             credential:
               name: wasabi-secret
               key: cloud
             config:
               region: eu-central-2
-              s3ForcePathStyle: true
               s3Url: https://s3.eu-central-2.wasabisys.com
-        # https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/volumesnapshotlocation.md
-        # https://velero.io/docs/v1.12/api-types/volumesnapshotlocation/#parameter-reference
-        volumeSnapshotLocation:
-          - name: wasabi
-            provider: openebs.io/cstor-blockstore
-            bucket: snelcom-velero
-            default: true
-            credential:
-              name: wasabi-secret
-              key: cloud
-            config:
-              namespace: mayastor
-              local: "true"
       credentials:
         name: wasabi-secret
         secretContents:
@@ -79,3 +67,21 @@
             [default]
             aws_access_key_id={{ velero_s3_access_key_id }}
             aws_secret_access_key={{ velero_s3_secret_access_key }}
+
+- name: Velero backup schedule
+  delegate_to: "{{ kubectl_host }}"
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    resource_definition:
+      apiVersion: velero.io/v1
+      kind: Schedule
+      metadata:
+        name: daily
+        namespace: velero
+      spec:
+        # UTC time
+        schedule: 41 12 * * *
+        template:
+          includedNamespaces:
+            - '*'
+          ttl: 720h