overhauls of most service docs

active/systemd_borg/backup.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Runs backup script for {{ repo_name }}
+After=network.target
+Wants=network-online.target
+
+[Service]
+Restart=no
+Type=oneshot
+ExecStart=/usr/local/script/backup-{{ repo_name }}.sh
+
+[Install]
+WantedBy=multi-user.target

active/systemd_borg/backup.sh.j2

@@ -33,6 +33,7 @@ borg create                         \
     --exclude-caches                \
     --exclude 'home/*/.cache/*'     \
     --exclude 'var/tmp/*'           \
+    --exclude 'home/*/.snapshots/*' \
 {% for dir in exclude_dirs %}
     --exclude '{{ dir }}' \
 {% endfor %}

active/systemd_borg/backup.timer

@@ -1,11 +1,11 @@
 [Unit]
-Description=Run Backup backup_{{ repo_name }}.service every hour
+Description=Run Backup backup-{{ repo_name }}.service every day
 
 [Timer]
-OnCalendar=hourly
+OnCalendar=*-*-* 2:00:00
 AccuracySec=10min
 Persistent=true
-Unit=ddns.{{ item.record }}.service
+Unit=backup-{{ repo_name }}.service
 
 [Install]
 WantedBy=timers.target

active/systemd_borg/borg.md

@@ -1,13 +1,14 @@
 # Borg Backup
 
 - [Borg Backup](#borg-backup)
-  - [Server Setup](#server-setup)
-    - [Adding a Client](#adding-a-client)
-    - [Installing the Backup Service](#installing-the-backup-service)
-      - [Adding Nextcloud](#adding-nextcloud)
-  - [Manual Client Use](#manual-client-use)
+  - [Install Borg](#install-borg)
+  - [Set up a new root client](#set-up-a-new-root-client)
+  - [Create a Backup Service](#create-a-backup-service)
+  - [Check backup service logs](#check-backup-service-logs)
+  - [Run a Manual Backup](#run-a-manual-backup)
+    - [Back up and Entire System](#back-up-and-entire-system)
 
-## Server Setup
+## Install Borg
 
 <https://borgbackup.readthedocs.io/en/stable/deployment/central-backup-server.html#user-and-group>
 
@@ -28,14 +29,16 @@ touch /home/backup/.ssh/authorized_keys
 chown -R backup:backup /home/backup/.ssh
 ```
 
-### Adding a Client
+## Set up a new root client
 
-Note: See [adding nextcloud](#adding-nextcloud) for nextcloud instructions here.
+Backups will be run as the root user. Generate them an SSH key to 
+
+On the server as root:
 
 ```bash
-export BACKUP_HOST=""
+export BACKUP_HOST="borg.reeselink.com"
 
-ssh-keygen -C ${USER}@${HOSTNAME} -f ~/.ssh/id_${BACKUP_HOST}
+ssh-keygen -C root@${HOSTNAME} -f ~/.ssh/id_${BACKUP_HOST}
 
 cat <<EOF >> ~/.ssh/config
 Host ${BACKUP_HOST}
@@ -43,39 +46,20 @@ Host ${BACKUP_HOST}
     IdentityFile ~/.ssh/id_${BACKUP_HOST}
     User backup
     Port 22
-    KeepAlive yes
 EOF
 ```
 
-Now on the server:
+Now on borg.reeselink.com as root:
 
 ```bash
-export CLIENT_FQDN=""
-# Should look like ssh-rsa abcd1234 backup@fqdn.something.com
-export SSH_PUBKEY=""
-export AUTHKEY_ENTRY="command=\"cd /home/backup/repos/${CLIENT_FQDN}; borg serve --restrict-to-path /home/backup/repos/${CLIENT_FQDN}\",restrict ${SSH_PUBKEY}"
+export CLIENT_FQDN="fqdn.reeseapps.com"
+export SSH_PUBKEY="ssh-rsa abcd1234 backup@fqdn.something.com"
+export BORG_COMMAND="cd /home/backup/repos/${CLIENT_FQDN}; borg serve --restrict-to-path /home/backup/repos/${CLIENT_FQDN}"
+export AUTHKEY_ENTRY="command=\"${BORG_COMMAND}\",restrict ${SSH_PUBKEY}"
 echo $AUTHKEY_ENTRY >> /home/backup/.ssh/authorized_keys
-
-mkdir /home/backup/repos/${CLIENT_FQDN}
-chown backup:backup /home/backup/repos/${CLIENT_FQDN}
 ```
 
-Then back on the client:
-
-```bash
-ssh borg.reeselink.com
-
-# root
-borg init --encryption none backup@${BACKUP_HOST}:root
-# home
-borg init --encryption none backup@${BACKUP_HOST}:home
-# app
-borg init --encryption none backup@${BACKUP_HOST}:gitea
-# another app
-borg init --encryption none backup@${BACKUP_HOST}:nextcloud
-```
-
-### Installing the Backup Service
+## Create a Backup Service
 
 Create your vars file in `secrets/host_vars.yaml`
 
@@ -97,30 +81,29 @@ stop_user_services:
 ```
 
 ```bash
+# Update all existing backup services for podman
+for var_file in $(ls active/systemd_borg/secrets); do
 ansible-playbook \
--i active/ansible/inventory.yaml \
+-i ansible/inventory.yaml \
 -l podman \
 active/systemd_borg/install_backup.yaml \
--e "@active/systemd_borg/secrets/gitea_vars.yaml"
+-e "@active/systemd_borg/secrets/$var_file"
+done
 ```
 
-#### Adding Nextcloud
+## Check backup service logs
 
-Rather than creating a client, just set the borg backup location to:
-
-```text
-backup@borg.reeselink.com:nextcloud
+```bash
+ssh podman journalctl -u 'backup-*' -f
 ```
 
-Then run the backup. It will generate a pubkey. Copy this into the authorized_keys file.
-
-## Manual Client Use
+## Run a Manual Backup
 
 ```bash
 borg list borg.reeselink.com:home
 
 # Do not include the first / in the path
-export PATH_TO_BACKUP=var/home/ducoterra
+export PATH_TO_BACKUP=home/ducoterra
 export BORG_REPO=borg.reeselink.com:home
 
 # If not initialized, do that now
@@ -148,6 +131,10 @@ borg create \
 -e "pp:/${PATH_TO_BACKUP}/.config/libvirt" \
 -e "pp:/${PATH_TO_BACKUP}/.local/share/containers" \
 -e "pp:/${PATH_TO_BACKUP}/.local/share/docker" \
+-e "pp:/${PATH_TO_BACKUP}/.npm" \
+-e "pp:/${PATH_TO_BACKUP}/.ollama" \
+-e "pp:/${PATH_TO_BACKUP}/Downloads" \
+-e "pp:/${PATH_TO_BACKUP}/Nextcloud" \
 ${BORG_REPO}::$(date +"%F-%H-%M-%S") \
 /${PATH_TO_BACKUP}
 
@@ -156,4 +143,31 @@ borg mount $BORG_REPO::2025-05-14-00-44-05 /mnt/
 
 # Restore a borg archive to a location (dry run)
 borg extract --dry-run --list --strip-components 1 $BORG_REPO::my-files home/USERNAME
-```
+```
+
+### Back up and Entire System
+
+```bash
+export BORG_REPO=borg.reeselink.com:root
+
+borg create \
+--verbose \
+--filter AME \
+--list \
+--stats \
+--progress \
+--show-rc \
+--compression lz4 \
+--exclude root/.cache \
+--exclude var/lib/docker \
+--exclude var/lib/containers \
+--exclude usr/share/ollama \
+--exclude home \
+--exclude proc \
+--exclude dev \
+--exclude sys \
+--exclude tmp \
+--exclude .snapshots \
+${BORG_REPO}::$(date +"%F-%H-%M-%S") \
+/
+```

active/systemd_borg/install_backup.yaml

@@ -8,36 +8,32 @@
       path: /usr/local/script
       state: directory
       mode: '0755'
-  - name: Copy backup.service
+  - name: Copy backup-{{ repo_name }}.service
     template:
       src: backup.service
       dest: /etc/systemd/system/backup-{{ repo_name }}.service
       owner: root
       group: root
       mode: '0644'
-  - name: Copy backup.timer
+  - name: Copy backup-{{ repo_name }}.timer
     template:
       src: backup.timer
       dest: /etc/systemd/system/backup-{{ repo_name }}.timer
       owner: root
       group: root
       mode: '0644'
-  - name: Template backup.sh
+  - name: Template backup-{{ repo_name }}.sh
     template:
       src: backup.sh.j2
       dest: /usr/local/script/backup-{{ repo_name }}.sh
       owner: root
       group: root
       mode: '0744'
-  # - name: Reload ddns timer
-  #   ansible.builtin.systemd_service:
-  #     state: restarted
-  #     name: ddns.{{ item.record }}.timer
-  #     enabled: true
-  #     daemon_reload: true
-  #   loop: "{{ records }}"
-  # - name: Run ddns service
-  #   ansible.builtin.systemd_service:
-  #     state: restarted
-  #     name: ddns.{{ item.record }}.service
-  #   loop: "{{ records }}"
+  - name: Initialize Repo {{ borg_user }}@{{ borg_host }}:{{ repo_name }}
+    script: /usr/bin/borg init -e none {{ borg_user }}@{{ borg_host }}:{{ repo_name }}
+    ignore_errors: yes
+  - name: Reload backup timer backup-{{ repo_name }}.timer
+    ansible.builtin.systemd_service:
+      name: backup-{{ repo_name }}.timer
+      enabled: true
+      daemon_reload: true