diff --git a/active/device_driveripper/btrfs-monitor.service b/active/device_driveripper/btrfs-monitor.service new file mode 100644 index 0000000..50af009 --- /dev/null +++ b/active/device_driveripper/btrfs-monitor.service @@ -0,0 +1,12 @@ +[Unit] +Description=Runs btrfs monitor +After=network.target +Wants=network-online.target + +[Service] +Restart=no +Type=oneshot +ExecStart=/root/scripts/btrfs-monitor.sh + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/active/device_driveripper/btrfs-monitor.sh b/active/device_driveripper/btrfs-monitor.sh new file mode 100644 index 0000000..59703c2 --- /dev/null +++ b/active/device_driveripper/btrfs-monitor.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash /usr/local/scripts/email-disk-stats.sh + +bash /usr/local/scripts/email-filesystem-stats.sh \ No newline at end of file diff --git a/active/device_driveripper/btrfs-monitor.timer b/active/device_driveripper/btrfs-monitor.timer new file mode 100644 index 0000000..1b00e09 --- /dev/null +++ b/active/device_driveripper/btrfs-monitor.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run btrfs monitoring script every day + +[Timer] +OnCalendar=*-*-* 8:00:00 +AccuracySec=10min +Persistent=true +Unit=btrfs-monitor.service + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/active/device_driveripper/driveripper.md b/active/device_driveripper/driveripper.md new file mode 100644 index 0000000..3c73cba --- /dev/null +++ b/active/device_driveripper/driveripper.md @@ -0,0 +1,219 @@ +# Driveripper + +- [Driveripper](#driveripper) + - [General Principles](#general-principles) + - [Important Locations](#important-locations) + - [Monitoring Scripts](#monitoring-scripts) + - [Quick Ansible Commands](#quick-ansible-commands) + - [Disk Mounts](#disk-mounts) + - [Disk Performance Testing](#disk-performance-testing) + - [General VM Notes](#general-vm-notes) + - [Best Practices](#best-practices) + - [OSBuild Composer](#osbuild-composer) + - [Retired Disks](#retired-disks) + - [Sending emails](#sending-emails) + +## General Principles + +1. Ingress: DDNS, HTTP, TLS, TCP/UDP Proxy +2. Backup: Borg, Storage, etc +3. Monitor: Glances, Emails, Scrubs, Defragmentation, PMLogger, etc. +4. Access: SSH, Cockpit +5. Virtualize: Libvirtd + +## Important Locations + +- `/etc/luks-keys`: luks keys +- `/usr/local/scripts`: admin scripts + +## Monitoring Scripts + +```bash +scp active/device_driveripper/btrfs-monitor.service driveripper:/etc/systemd/system/ +scp active/device_driveripper/btrfs-monitor.timer driveripper:/etc/systemd/system/ + +systemctl enable --now btrfs-monitor.timer +``` + +## Quick Ansible Commands + +```bash +# Update Caddy +ansible-playbook -i ansible/inventory.yaml -l proxy active/podman_caddy/install_caddy.yaml + +# Update DDNS +ansible-playbook -i ansible/inventory.yaml -l proxy active/podman_ddns/install_ddns.yaml +``` + +## Disk Mounts + +1. All btrfs `subvolid=5` volumes should be mounted under `/btrfs` +1. Since `/media` is for removable media, and `/mnt` is for temporary mounts, all + btrfs subvolumes should be mounted in `/srv` for consistency. + +## Disk Performance Testing + +```bash +# Write +dd if=/dev/zero of=test.dat bs=1M count=10000 conv=fsync status=progress + +# Read +dd if=test.dat of=/dev/null bs=1M count=10000 conv=fsync status=progress +``` + +## General VM Notes + +1. Use ext4 as your filesystem within the VM to cut down on multi COW +2. Use raw images (convert from qcow2 where possible) to cut down on multi COW +3. Create a folder at `/srv/vm/boot` for boot disks +4. Create a folder at `/srv/vm/data` for data disks + +### Best Practices + +1. Remove the default bridge: `virsh net-undefined default` + +## OSBuild Composer + +1. Blueprints are stored in /srv/smb/ducoterra/images/blueprints +2. Builds are stored in /srv/smb/ducoterra/images/builds + +```bash +# Push a blueprint +composer-cli blueprints push /srv/smb/ducoterra/images/blueprints/fedora43-base.toml + +# Start a build +composer-cli compose start fedora43-base qcow2 + +# Watch the build +watch composer-cli compose list running + +# Remove the old image +mv /srv/smb/ducoterra/images/builds/fedora-43-base.qcow2 /srv/smb/ducoterra/images/builds/fedora-43-base-old.qcow2 + +# Extract the build +composer-cli compose image \ +--filename /srv/smb/ducoterra/images/builds/fedora-43-base.qcow2 \ +92a2f1dd-9238-4d50-ab1d-cd7c8e355cac + +# Convert the build to raw +qemu-img convert -f qcow2 -O raw \ +/srv/smb/pool0/ducoterra/images/builds/fedora-43-base.qcow2 \ +/srv/vm/pool1/fedora-43-base.raw + +# Install (Change password for default user ducoterra!) +virt-install \ +--name "gitlab" \ +--boot uefi,firmware.feature0.name=secure-boot,firmware.feature0.enabled=no \ +--cpu host-passthrough --vcpus sockets=1,cores=8,threads=2 \ +--ram=8192 \ +--os-variant=fedora41 \ +--network bridge:bridge0 \ +--graphics none \ +--console pty,target.type=virtio \ +--import --disk "path=/srv/vm/pool1/gitlab.raw,bus=virtio" + +# convert a cloud-init image to raw +qemu-img convert -f qcow2 -O raw \ +/srv/smb/ducoterra/images/cloud/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2 \ +/srv/vm/pool1/fedora-43-cloud.raw + +# Resize +qemu-img resize -f raw /srv/vm/pool1/fedora-43-cloud.raw 128G + +# Install with cloud-init +virt-install \ +--name "freeipa" \ +--boot uefi,firmware.feature0.name=secure-boot,firmware.feature0.enabled=no \ +--cpu host-passthrough --vcpus sockets=1,cores=8,threads=2 \ +--ram=8192 \ +--os-variant=fedora41 \ +--network bridge:bridge0 \ +--graphics none \ +--import --disk "path=/srv/vm/pool1/freeipa-boot.raw,bus=virtio" \ +--cloud-init disable=yes,user-data="/srv/smb/ducoterra/images/cloud-init/fedora/cloud-init/user-data,meta-data=/srv/smb/ducoterra/images/cloud-init/fedora/cloud-init/meta-data" + +# Reattach to console if needed +virsh console fedora-43-base-test + +passwd ducoterra +hostnamectl hostname + +# Attaching an extra drive +fdisk /dev/vdb +> g +> n +> enter enter enter +> w + +mkfs.ext4 /dev/vdb1 +lsblk -fs # grab UUID +vim /etc/fstab +systemctl daemon-reload +mount -a --mkdir +``` + +## Retired Disks + +Retired 12-19-2025 + +![alt text](image.png) + +Retired 12-19-2025 + +![alt text](image-1.png) + +### Sending emails + +```bash +# s-nail is mailx +dnf install -y msmtp +vim /etc/ssmtp/ssmtp.conf +``` + +Fill out the configuration like this: + +```conf +# A system wide configuration file is optional. +# If it exists, it usually defines a default account. +# This allows msmtp to be used like /usr/sbin/sendmail. +account default + +# The SMTP smarthost +host stmp-server.com + +# Use TLS on port 465. On this port, TLS starts without STARTTLS. +port 465 +tls on +tls_starttls off + +# Construct envelope-from addresses of the form "user@oursite.example" +from driveripper@reeselink.com +# Do not allow programs to override this envelope-from address via -f +allow_from_override off +# Always set a From header that matches the envelope-from address +set_from_header on + +# Syslog logging with facility LOG_MAIL instead of the default LOG_USER +syslog LOG_MAIL + +# User info +auth on +user username +password password +``` + +Send an email like this: + +```bash +# Basic message +printf "Subject: Test\n\nhello there username." | msmtp admin@ducoterra.net + +# Or some btrfs stats +printf "Subject: Pool0 Stats\n\n$(btrfs device stats /btrfs/pool0)" | msmtp admin@ducoterra.net + +# Or some more btrfs stats +printf "Subject: Pool0 Stats\n\n$(btrfs filesystem show)" | msmtp admin@ducoterra.net + +# Or some smartmontools stats +printf "To: admin@ducoterra.net\n\nFrom: driveripper@reeselink.com\n\nSubject: Disk Stats\n\n$(smartctl -a /dev/sda | grep "SMART Attributes" -A 18)" | msmtp admin@ducoterra.net +``` \ No newline at end of file diff --git a/active/device_driveripper/email-disk-stats.sh b/active/device_driveripper/email-disk-stats.sh new file mode 100644 index 0000000..258d98d --- /dev/null +++ b/active/device_driveripper/email-disk-stats.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +export EMAIL_FILE=/root/disk-stats-email.txt +export EMAIL_TO=admin@ducoterra.net + +printf "Subject: Driveripper Disk Stats\n\nTo: ${EMAIL_TO}\n\n" > ${EMAIL_FILE} + +printf "Pool 0\n" >> ${EMAIL_FILE} +btrfs device stats /btrfs/pool0 >> ${EMAIL_FILE} + +printf "\nPool 1\n" >> ${EMAIL_FILE} +btrfs device stats /btrfs/pool1 >> ${EMAIL_FILE} + +printf "\nBackup 0\n" >> ${EMAIL_FILE} +btrfs device stats /btrfs/backup0 >> ${EMAIL_FILE} + +cat ${EMAIL_FILE} | msmtp ${EMAIL_TO} \ No newline at end of file diff --git a/active/device_driveripper/email-filesystem-stats.sh b/active/device_driveripper/email-filesystem-stats.sh new file mode 100644 index 0000000..9fe60c4 --- /dev/null +++ b/active/device_driveripper/email-filesystem-stats.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +export EMAIL_FILE=/root/filesystem-usage-email.txt +export EMAIL_TO=admin@ducoterra.net + +printf "Subject: Driveripper Filesystem Usage\n\nTo: ${EMAIL_TO}\n\n" > ${EMAIL_FILE} + +printf "Pool 0\n" >> ${EMAIL_FILE} +btrfs filesystem usage /btrfs/pool0 >> ${EMAIL_FILE} + +printf "\nPool 1\n" >> ${EMAIL_FILE} +btrfs filesystem usage /btrfs/pool1 >> ${EMAIL_FILE} + +printf "\nBackup 0\n" >> ${EMAIL_FILE} +btrfs filesystem usage /btrfs/backup0 >> ${EMAIL_FILE} + +cat ${EMAIL_FILE} | msmtp ${EMAIL_TO} \ No newline at end of file diff --git a/active/device_driveripper/image-1.png b/active/device_driveripper/image-1.png new file mode 100644 index 0000000..179857c Binary files /dev/null and b/active/device_driveripper/image-1.png differ diff --git a/active/device_driveripper/image.png b/active/device_driveripper/image.png new file mode 100644 index 0000000..9bc8c29 Binary files /dev/null and b/active/device_driveripper/image.png differ