From 8abfbcf561966422941cc402b0277f0096a50cf0 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Tue, 1 Feb 2022 20:25:26 -0500 Subject: [PATCH] Fix issues where cryptsetup would fail to unmount There were occasions where removing the backup drive would confuse luks and prevent re-mounting. This fixes the issue by attempting to luks close and open the drive before mounting, thereby correcting any issues where the drive would stick. --- README.md | 8 ++ .../arch_backup/files/scripts/btrfs_backup.sh | 82 -------------- ansible/arch_backup/tasks/main.yml | 23 ++-- .../arch_backup/templates/btrfs_backup.sh.j2 | 101 ++++++++++++++++++ ansible/arch_backup/vars/main.yml | 8 ++ 5 files changed, 128 insertions(+), 94 deletions(-) delete mode 100755 ansible/arch_backup/files/scripts/btrfs_backup.sh create mode 100755 ansible/arch_backup/templates/btrfs_backup.sh.j2 diff --git a/README.md b/README.md index ec2d02d..95cc8df 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ ansible-playbook -i hosts --ask-become-pass playbooks/pi.yaml Run the manjaro playbook +NOTE: Restore home directory and __REBOOT__ first + ```bash ansible-playbook --ask-become-pass ansible/setup-full.yml ``` @@ -92,4 +94,10 @@ sudo mkdir -p /mnt/backup0 # Add to fstab echo '/dev/mapper/backup0 /mnt/backup0 btrfs defaults,noatime,compress=zstd 0 0' > /etc/fstab + +# mount +sudo cryptsetup luksOpen /dev/disk/by-uuid/1d7ce570-e695-47a0-9dda-5f14b5b20e21 backup0 --key-file=/home/ducoterra/.lukskeys/backup0 + +# close (or fix issues) +sudo cryptsetup luksClose backup0 ``` diff --git a/ansible/arch_backup/files/scripts/btrfs_backup.sh b/ansible/arch_backup/files/scripts/btrfs_backup.sh deleted file mode 100755 index 7f7c68c..0000000 --- a/ansible/arch_backup/files/scripts/btrfs_backup.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -export SOURCE_DIR=${SOURCE_DIR:=/} -# Fix basename / showing up as "/" -> change to "root" -if [ $(basename $SOURCE_DIR) = / ]; then - export SNAPSHOT_PREFIX=${SNAPSHOT_PREFIX:=root} -else - export SNAPSHOT_PREFIX=${SNAPSHOT_PREFIX:=$(basename $SOURCE_DIR)} -fi -# Set snapshot prefix based on basename -export SNAPSHOT_TIME=$(date +"%y_%m_%d-%H.%M") -export SNAPSHOT_NAME=$SNAPSHOT_PREFIX-$SNAPSHOT_TIME -export SNAPSHOT_DIR=${SNAPSHOT_DIR:=/.snapshots} -export LATEST=$SNAPSHOT_PREFIX-latest -export BACKUP_DRIVE_MNT=${BACKUP_DRIVE_MNT:=/mnt/backup0} -export BACKUP_DIR=${BACKUP_DIR:=/mnt/backup0/DucoBacktop} - -# Show snapshot settings -echo "SOURCE_DIR: $SOURCE_DIR" -echo "SNAPSHOT_PREFIX: $SNAPSHOT_PREFIX" -echo "SNAPSHOT_TIME: $SNAPSHOT_TIME" -echo "SNAPSHOT_NAME: $SNAPSHOT_NAME" -echo "SNAPSHOT_DIR: $SNAPSHOT_DIR" -echo "LATEST: $LATEST" -echo "BACKUP_DRIVE_MNT: $BACKUP_DRIVE_MNT" -echo "BACKUP_DIR: $BACKUP_DIR" - -# Sync latest backups -# The "latest" symlinks can get out of sync for a variety of reasons, -# including backups run while the disk is unplugged. -# They must be kept in sync so that snapshots are sent with a parent -# that actually exists on the backup drive -# -# In order to keep them in sync we'll do the following: -# 1. Check the -latest on the backup drive matches -# -latest in the snapshot directory -# 2. If they don't match, assume the backup drive has the correct -# snapshot. Replace the -latest symlink in the snapshots -# directory with the one from the backup drive -# -# First check if the symlinks exist -if [ -L $SNAPSHOT_DIR/$LATEST ] && [ -L $BACKUP_DIR/$LATEST ] -then - # Get the actual name of the latest backup - LATEST_SNAPSHOT=$(basename $(readlink $SNAPSHOT_DIR/$LATEST)) - LATEST_BACKUP=$(basename $(readlink $BACKUP_DIR/$LATEST)) - # If the latest backups don't match - if [ $LATEST_SNAPSHOT != $LATEST_BACKUP ] - then - echo "Detected drift. Synchronizing latest snapshot with backup. Set to $LATEST_BACKUP." - # Remove and replace the snapshot directory's latest - rm $SNAPSHOT_DIR/$LATEST - ln -s $SNAPSHOT_DIR/$LATEST_BACKUP $SNAPSHOT_DIR/$LATEST - fi -fi - -# Create readonly snapshot -btrfs subvolume snapshot -r $SOURCE_DIR $SNAPSHOT_DIR/$SNAPSHOT_NAME - -# Attempt to mount backup disk -mount $BACKUP_DRIVE_MNT - -# Check if backup disk is mounted -mountpoint $BACKUP_DRIVE_MNT -if [ $? = 0 ]; then # backup drive mounted - # If we have a latest, use it as the parent - if [ -L $BACKUP_DIR/$LATEST ]; then - btrfs send -p $SNAPSHOT_DIR/$LATEST $SNAPSHOT_DIR/$SNAPSHOT_NAME | btrfs receive $BACKUP_DIR - else - btrfs send $SNAPSHOT_DIR/$SNAPSHOT_NAME | btrfs receive $BACKUP_DIR - fi - - # Update latest in backup dir - rm -f $BACKUP_DIR/$LATEST - ln -s $BACKUP_DIR/$SNAPSHOT_NAME $BACKUP_DIR/$LATEST -else - echo "Backup location $BACKUP_DRIVE_MNT not mounted. Snapshot $SNAPSHOT_NAME not synced" -fi - -# Update latest in snapshot dir -rm -f $SNAPSHOT_DIR/$LATEST -ln -s $SNAPSHOT_DIR/$SNAPSHOT_NAME $SNAPSHOT_DIR/$LATEST diff --git a/ansible/arch_backup/tasks/main.yml b/ansible/arch_backup/tasks/main.yml index 9691276..4bc75e9 100644 --- a/ansible/arch_backup/tasks/main.yml +++ b/ansible/arch_backup/tasks/main.yml @@ -1,11 +1,15 @@ --- # Backup -- name: Create backup mount directory +- name: Ensure snapshot directory + file: + state: directory + path: "{{ snapshots.path }}" + become: yes +- name: Ensure backup mount directory file: state: directory path: "{{ mount.path }}" become: yes - tags: backup - name: Ensure {{ mount.path }} device exists in crypttab community.general.crypttab: name: "{{ disk.name }}" @@ -15,7 +19,6 @@ state: present become: yes no_log: true - tags: backup - name: Ensure {{ disk.name }} mount exists in fstab ansible.posix.mount: path: "{{ mount.path }}" @@ -24,29 +27,25 @@ opts: nofail,x-systemd.device-timeout=1,noatime,compress=zstd state: present become: yes - tags: backup - name: Ensure /usr/local/scripts exists file: state: directory path: '/usr/local/scripts' become: yes - tags: backup -- name: Copy btrfs_backup.sh - ansible.builtin.copy: - src: scripts/btrfs_backup.sh +- name: Template btrfs_backup.sh + ansible.builtin.template: + src: btrfs_backup.sh.j2 dest: /usr/local/scripts/btrfs_backup.sh owner: root group: root - mode: '0770' + mode: '0744' become: yes - tags: backup - name: Ensure hourly backups of each item in backups ansible.builtin.cron: name: "hourly backup of {{ item }}" minute: "0" job: "export SOURCE_DIR={{ item }}; /usr/local/scripts/btrfs_backup.sh" become: yes - tags: backup loop: "{{ backups }}" - name: Ensure cronie service started ansible.builtin.systemd: @@ -55,4 +54,4 @@ daemon_reload: yes enabled: yes become: yes - tags: backup + diff --git a/ansible/arch_backup/templates/btrfs_backup.sh.j2 b/ansible/arch_backup/templates/btrfs_backup.sh.j2 new file mode 100755 index 0000000..1e5d0f1 --- /dev/null +++ b/ansible/arch_backup/templates/btrfs_backup.sh.j2 @@ -0,0 +1,101 @@ +#!/bin/bash + +# Backup info +export BACKUP_DRIVE_UUID={{ disk.uuid }} +export BACKUP_DRIVE={{ disk.name }} +export BACKUP_DRIVE_PASSWORD={{ disk.password }} + +# For notifications +export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{{ notifications.user.uid }}/bus +export USER={{ notifications.user.name }} + +export SOURCE_DIR=${SOURCE_DIR:=/} +# Fix basename / showing up as "/" -> change to "root" +if [ $(basename $SOURCE_DIR) = / ]; then + export SNAPSHOT_PREFIX=${SNAPSHOT_PREFIX:=root} +else + export SNAPSHOT_PREFIX=${SNAPSHOT_PREFIX:=$(basename $SOURCE_DIR)} +fi +# Set snapshot prefix based on basename +export SNAPSHOT_TIME=$(date +"%y_%m_%d-%H.%M") +export SNAPSHOT_NAME=$SNAPSHOT_PREFIX-$SNAPSHOT_TIME +export SNAPSHOT_DIR=${SNAPSHOT_DIR:=/.snapshots} +export LATEST=$SNAPSHOT_PREFIX-latest +export BACKUP_DRIVE=backup0 +export BACKUP_DRIVE_MNT=${BACKUP_DRIVE_MNT:=/mnt/$BACKUP_DRIVE} +export BACKUP_DIR=${BACKUP_DIR:=/mnt/$BACKUP_DRIVE/$(hostname)} + +# Show snapshot settings +echo "SOURCE_DIR: $SOURCE_DIR" +echo "SNAPSHOT_PREFIX: $SNAPSHOT_PREFIX" +echo "SNAPSHOT_TIME: $SNAPSHOT_TIME" +echo "SNAPSHOT_NAME: $SNAPSHOT_NAME" +echo "SNAPSHOT_DIR: $SNAPSHOT_DIR" +echo "LATEST: $LATEST" +echo "BACKUP_DRIVE_MNT: $BACKUP_DRIVE_MNT" +echo "BACKUP_DIR: $BACKUP_DIR" + +# Create readonly snapshot +btrfs subvolume snapshot -r $SOURCE_DIR $SNAPSHOT_DIR/$SNAPSHOT_NAME + +# Attempt to mount backup disk +# Check if backup disk is mounted +mountpoint $BACKUP_DRIVE_MNT +if [ $? != 0 ]; then + cryptsetup luksClose $BACKUP_DRIVE + cryptsetup luksOpen /dev/disk/by-uuid/$BACKUP_DRIVE_UUID $BACKUP_DRIVE --key-file=$BACKUP_DRIVE_PASSWORD + mount $BACKUP_DRIVE_MNT +fi + +# Check if backup disk is mounted +mountpoint $BACKUP_DRIVE_MNT +if [ $? = 0 ]; then # backup drive mounted + if [ ! -d $BACKUP_DIR ]; then + mkdir -p $BACKUP_DIR + fi + # Sync latest backups + # The "latest" symlinks can get out of sync for a variety of reasons, + # including backups run while the disk is unplugged. + # They must be kept in sync so that snapshots are sent with a parent + # that actually exists on the backup drive + # + # In order to keep them in sync we'll do the following: + # 1. Check the -latest on the backup drive matches + # -latest in the snapshot directory + # 2. If they don't match, assume the backup drive has the correct + # snapshot. Replace the -latest symlink in the snapshots + # directory with the one from the backup drive + # + # First check if the symlinks exist + if [ -L $SNAPSHOT_DIR/$LATEST ] && [ -L $BACKUP_DIR/$LATEST ]; then + # Get the actual name of the latest backup + LATEST_SNAPSHOT=$(basename $(readlink $SNAPSHOT_DIR/$LATEST)) + LATEST_BACKUP=$(basename $(readlink $BACKUP_DIR/$LATEST)) + # If the latest backups don't match + if [ $LATEST_SNAPSHOT != $LATEST_BACKUP ] + then + echo "Detected drift. Synchronizing latest snapshot with backup. Set to $LATEST_BACKUP." + # Remove and replace the snapshot directory's latest + rm $SNAPSHOT_DIR/$LATEST + ln -s $SNAPSHOT_DIR/$LATEST_BACKUP $SNAPSHOT_DIR/$LATEST + fi + fi + + # If we have a latest, use it as the parent + if [ -L $BACKUP_DIR/$LATEST ]; then + btrfs send -p $SNAPSHOT_DIR/$LATEST $SNAPSHOT_DIR/$SNAPSHOT_NAME | btrfs receive $BACKUP_DIR + else + btrfs send $SNAPSHOT_DIR/$SNAPSHOT_NAME | btrfs receive $BACKUP_DIR + fi + + # Update latest in backup dir + rm -f $BACKUP_DIR/$LATEST + ln -s $BACKUP_DIR/$SNAPSHOT_NAME $BACKUP_DIR/$LATEST + sudo -E -u $USER notify-send "Backup complete" "Snapshot $SNAPSHOT_NAME completed successfully" +else + sudo -E -u $USER notify-send "Backup failed" "$BACKUP_DRIVE_MNT not mounted. Snapshot $SNAPSHOT_NAME not synced" -u critical +fi + +# Update latest in snapshot dir +rm -f $SNAPSHOT_DIR/$LATEST +ln -s $SNAPSHOT_DIR/$SNAPSHOT_NAME $SNAPSHOT_DIR/$LATEST diff --git a/ansible/arch_backup/vars/main.yml b/ansible/arch_backup/vars/main.yml index a390512..a89d09d 100644 --- a/ansible/arch_backup/vars/main.yml +++ b/ansible/arch_backup/vars/main.yml @@ -2,6 +2,9 @@ mount: path: /mnt/backup0 +snapshots: + path: /.snapshots + disk: name: backup0 uuid: 1d7ce570-e695-47a0-9dda-5f14b5b20e21 @@ -10,3 +13,8 @@ disk: backups: - / - /home + +notifications: + user: + name: ducoterra + uid: 1000