#!/bin/bash # backup-watch: email this VM's owner when a restic repository here has had # no new snapshot for a while, or when the disk is filling up. # # It runs on the backup VM and looks only at file times and disk usage. # It never needs a repository password, so it can't read your backups. # # backup-watch.sh check, and email the owner if something's wrong # backup-watch.sh --dry-run check and print, never email # # Settings (environment): REPOS (default /home/exedev/restic), # MAX_AGE_HOURS (default 48), MAX_DISK_PERCENT (default 80). set -euo pipefail REPOS=${REPOS:-/home/exedev/restic} MAX_AGE_HOURS=${MAX_AGE_HOURS:-48} MAX_DISK_PERCENT=${MAX_DISK_PERCENT:-80} problems="" fresh=0 now=$(date +%s) for repo in "$REPOS"/*/; do [ -f "$repo/config" ] || continue name=$(basename "$repo") newest=$(find "$repo/snapshots" -type f -printf '%T@\n' 2>/dev/null | sort -n | tail -n 1) if [ -z "$newest" ]; then problems+="$name: no snapshots yet"$'\n' continue fi age=$(( (now - ${newest%.*}) / 3600 )) if [ "$age" -ge "$MAX_AGE_HOURS" ]; then problems+="$name: last snapshot ${age} hours ago"$'\n' else fresh=$((fresh + 1)) fi done used=$(df --output=pcent "$REPOS" | tail -n 1 | tr -dc 0-9) if [ "$used" -ge "$MAX_DISK_PERCENT" ]; then free=$(df -h --output=avail "$REPOS" | tail -n 1 | tr -d ' ') problems+="disk: ${used}% used, ${free} free"$'\n' fi if [ -z "$problems" ]; then echo "ok: $fresh repositories with a snapshot in the last ${MAX_AGE_HOURS} hours, disk ${used}% used" exit 0 fi printf '%s' "$problems" [ "${1:-}" = "--dry-run" ] && exit 0 to=$(curl -fsS https://reflection.int.exe.xyz/email | jq -r .email) body="backup-watch on $(hostname) found: ${problems} A repository with no recent snapshot usually means that machine is off, its schedule stopped, or its exe.dev VM token expired or was removed. Run 'restic snapshots' there to see the error. A full disk stops every backup: grow it with 'ssh exe.dev resize $(hostname) --disk=...'." jq -n --arg to "$to" --arg subject "Backups on $(hostname) need a look" --arg body "$body" \ '{to: $to, subject: $subject, body: $body}' | curl -fsS -X POST http://169.254.169.254/gateway/email/send \ -H 'Content-Type: application/json' --data-binary @- >/dev/null echo "emailed $to"