#!/bin/bash
# alpha-lxd-pool-guard v2 - keeps LXD btrfs pools mountable and grows them with the server.
# Source: provisioning/helpers/alpha-lxd-pool-guard (installed by steps/01-panel.sh). Timer: alpha-lxd-pool-guard.timer (every 2 min). Log: /var/log/alpha-lxd-pool-guard.log
# Usage: alpha-lxd-pool-guard [--dry-run] [--threshold PCT] [--reserve GIB]   (overrides are for testing)
set -u
LOG=/var/log/alpha-lxd-pool-guard.log
DRY=0; OVR_THR=""; OVR_RES=""
while [ $# -gt 0 ]; do case "$1" in
  --dry-run) DRY=1;; --threshold) OVR_THR="$2"; shift;; --reserve) OVR_RES="$2"; shift;;
  *) echo "usage: $0 [--dry-run] [--threshold PCT] [--reserve GIB]" >&2; exit 2;; esac; shift; done
THRESHOLD_PCT=80; GROW_STEP_GIB=10; HOST_RESERVE_GIB=10; MAX_TOTAL_GIB=0
[ -r /etc/default/alpha-lxd-pool-guard ] && . /etc/default/alpha-lxd-pool-guard
[ -n "$OVR_THR" ] && THRESHOLD_PCT=$OVR_THR; [ -n "$OVR_RES" ] && HOST_RESERVE_GIB=$OVR_RES
GIB=1073741824
log(){ if [ "$DRY" = 1 ]; then echo "$*"; else echo "$(date -Is) $*" >> "$LOG"; fi; }
say(){ [ "$DRY" = 1 ] && echo "$*"; }

# ---- 1. btrfs must be available to the kernel (explicit modprobe ignores any blacklist)
if ! grep -qw btrfs /proc/filesystems; then
  if /usr/sbin/modprobe btrfs; then log "loaded btrfs kernel module"; else log "ERROR: modprobe btrfs failed"; fi
fi

# ---- 2. if anything re-creates a btrfs blacklist, disable it so the next boot is safe
for f in $(grep -rlsE '^[[:space:]]*blacklist[[:space:]]+btrfs([[:space:]]|$)' /etc/modprobe.d/ 2>/dev/null); do
  sed -i -E 's/^([[:space:]]*blacklist[[:space:]]+btrfs)/# disabled by alpha-lxd-pool-guard: \1/' "$f"
  log "neutralised btrfs blacklist in $f"
done

command -v lxc >/dev/null 2>&1 || exit 0
timeout 20 lxc query /1.0 >/dev/null 2>&1 || { say "LXD not answering; skip"; exit 0; }

# ---- 3. report pools LXD still marks UNAVAILABLE (LXD re-tries the mount itself every ~70s once the module exists)
bad=$(timeout 30 lxc storage list --format compact 2>/dev/null | awk 'NR>1 && toupper($NF)=="UNAVAILABLE"{print $1}' | tr '\n' ' ')
[ -n "$bad" ] && log "WARN: LXD pool(s) UNAVAILABLE: $bad - module present, waiting for LXD remount"

# ---- 4. elastic auto-grow for loop-backed btrfs pools
for pool in $(timeout 30 lxc storage list --format compact 2>/dev/null | awk 'NR>1 && $2=="btrfs" && $3 ~ /\.img$/ && toupper($NF)=="CREATED"{print $1}'); do
  src=$(timeout 20 lxc storage get "$pool" source 2>/dev/null); [ -n "$src" ] || continue
  res=$(timeout 20 lxc query "/1.0/storage-pools/$pool/resources" 2>/dev/null) || continue
  res=$(printf "%s" "$res" | tr -d "[:space:]")
  space=$(echo "$res" | grep -oE '"space":\{[^}]*\}')
  used=$(echo "$space" | grep -oE '"used":[0-9]+' | head -1 | cut -d: -f2)
  total=$(echo "$space" | grep -oE '"total":[0-9]+' | head -1 | cut -d: -f2)
  [ -n "$used" ] && [ -n "$total" ] && [ "$total" -gt 0 ] || continue
  pct=$(( used * 100 / total ))
  host_avail=$(df -B1 --output=avail "$(dirname "$src")" 2>/dev/null | tail -1 | tr -d ' ')
  [ -n "$host_avail" ] || continue
  # the pool may only promise what the host can really deliver
  max_safe=$(( used + host_avail - HOST_RESERVE_GIB * GIB ))
  [ "$MAX_TOTAL_GIB" -gt 0 ] && [ "$max_safe" -gt $(( MAX_TOTAL_GIB * GIB )) ] && max_safe=$(( MAX_TOTAL_GIB * GIB ))
  say "pool=$pool used=$((used/GIB))GiB total=$((total/GIB))GiB pct=$pct% host_avail=$((host_avail/GIB))GiB max_safe=$((max_safe/GIB))GiB"
  [ "$pct" -ge "$THRESHOLD_PCT" ] || continue
  want=$(( total + GROW_STEP_GIB * GIB ))
  [ "$want" -gt "$max_safe" ] && want=$max_safe
  want_gib=$(( want / GIB ))            # round down to whole GiB
  if [ "$want_gib" -le $(( total / GIB )) ]; then
    log "WARN: pool $pool is ${pct}% full (${total}B) and cannot grow: host free $((host_avail/GIB))GiB minus ${HOST_RESERVE_GIB}GiB reserve - add disk"
    continue
  fi
  if [ "$DRY" = 1 ]; then say "DRY-RUN: would run: lxc storage set $pool size=${want_gib}GiB"; continue; fi
  if out=$(timeout 120 lxc storage set "$pool" size="${want_gib}GiB" 2>&1); then
    log "grew pool $pool from $((total/GIB))GiB to ${want_gib}GiB (was ${pct}% full)"
  else
    log "ERROR: growing pool $pool to ${want_gib}GiB failed: $out"
  fi
done

# ---- 5. nothing a deleted instance owned may stay on the box
# Every 10 min: names with leftovers (pool subvolume, LXD devices/logs, console
# vhost + cert + app dir, panel-store rows) but no instance are purged once they
# have been orphaned for 10 min — catches `lxc delete` run outside the panel, and
# anything a crash left half-done. alpha-virt refuses unless the pool is mounted
# and LXD's instance list came back intact.
ORPHAN_EVERY_SEC=${ORPHAN_EVERY_SEC:-600}; ORPHAN_GRACE_SEC=${ORPHAN_GRACE_SEC:-600}
stamp=/var/lib/alpha-lxd/.orphan-sweep
if [ -x /usr/local/sbin/alpha-virt ]; then
  if [ "$DRY" = 1 ]; then
    /usr/local/sbin/alpha-virt lxd-orphans 2>&1 | sed 's/^/orphans: /'
  elif [ ! -f "$stamp" ] || [ $(( $(date +%s) - $(stat -c %Y "$stamp" 2>/dev/null || echo 0) )) -ge "$ORPHAN_EVERY_SEC" ]; then
    mkdir -p /var/lib/alpha-lxd; touch "$stamp"
    out=$(timeout 600 /usr/local/sbin/alpha-virt lxd-orphans --purge --grace "$ORPHAN_GRACE_SEC" 2>&1) || true
    printf '%s\n' "$out" | grep -E '^(PURGE|  )' | while IFS= read -r l; do log "orphan-sweep: $l"; done
  fi
fi
exit 0
