#!/bin/bash
# Per-minute CPU%, MEM%, and cumulative nginx request counter → ring buffer CSV.
# Columns: epoch,cpu_pct,mem_pct,nginx_requests_total
#
# Plus a DAILY DISK SNAPSHOT, taken by the first tick of each UTC day, so "storage spiked"
# is one lookup instead of a forensic `find -ctime` (2026-10-22: 52 → 57.7 GB took an hour to explain):
#   /var/lib/alpha-metrics/disk.csv             day,total_b,used_b,avail_b   (df /, one row a day)
#   /var/lib/alpha-metrics/disk/YYYY-MM-DD.tsv  bytes<TAB>dir   (du -x, ALLOCATED bytes, 4 levels, ≥ 50 MB)
#
#   alpha-metrics-sample disk-diff [days]   what grew or shrank since N days ago (default 4)
#   alpha-metrics-sample disk-snapshot      take today's snapshot now (replaces today's)
#
# ⚠️ du -x stays on /: loop-mounted site volumes count once, as their image file under
#    /var/lib/alpha-quota; the LXD btrfs pool counts as disks/alpha.img (allocated, not its 50 GB size).
D=/var/lib/alpha-metrics
F=$D/samples.csv
KEEP_DAYS=120

disk_snapshot() {
  mkdir -p "$D/disk" && chmod 755 "$D/disk"
  local day tmp tot used avail
  day=$(date -u +%F); tmp="$D/disk/.$day.tsv.tmp"
  nice -n 19 ionice -c3 du -x -B1 -d 4 -t 50M / 2>/dev/null | sort -t "$(printf '\t')" -k2 > "$tmp"
  [ -s "$tmp" ] || { rm -f "$tmp"; return 1; }
  mv "$tmp" "$D/disk/$day.tsv" && chmod 644 "$D/disk/$day.tsv"
  read -r tot used avail < <(df -B1 --output=size,used,avail / | tail -1)
  { [ -f "$D/disk.csv" ] && grep -v "^$day," "$D/disk.csv"; echo "$day,$tot,$used,$avail"; } | tail -n $KEEP_DAYS > "$D/disk.csv.tmp" \
    && mv "$D/disk.csv.tmp" "$D/disk.csv" && chmod 644 "$D/disk.csv"
  find "$D/disk" -name '*.tsv' -mtime +$KEEP_DAYS -delete
}

disk_diff() {
  local days=${1:-4} new old want
  [[ "$days" =~ ^[0-9]{1,3}$ ]] || { echo "usage: alpha-metrics-sample disk-diff [days]" >&2; exit 2; }
  new=$(ls "$D"/disk/*.tsv 2>/dev/null | tail -1)
  want="$D/disk/$(date -u -d "$days days ago" +%F).tsv"
  # the newest snapshot on or before that day; if history is shorter, the oldest there is
  old=$(ls "$D"/disk/*.tsv 2>/dev/null | awk -v w="$want" '$0 <= w' | tail -1)
  [ -z "$old" ] && old=$(ls "$D"/disk/*.tsv 2>/dev/null | head -1)
  if [ -z "$new" ] || [ "$old" = "$new" ]; then
    echo "Only $(ls "$D"/disk/*.tsv 2>/dev/null | wc -l) daily snapshot(s) so far — a diff needs two (one is taken every UTC day)."
    exit 1
  fi
  local a b; a=$(basename "$old" .tsv); b=$(basename "$new" .tsv)
  echo "Disk / from $a to $b"
  awk -F, -v a="$a" -v b="$b" '$1==a{u0=$3; v0=$4} $1==b{u1=$3; v1=$4; t=$2}
    END{ if (u0 != "" && u1 != "") printf "  used %.1f → %.1f GB (%+.2f GB)   available %.1f → %.1f GB   size %.1f GB\n", u0/1e9, u1/1e9, (u1-u0)/1e9, v0/1e9, v1/1e9, t/1e9 }' "$D/disk.csv"
  echo "Directories that changed by ≥ 100 MB (allocated; nested rows include their children):"
  awk -F'\t' 'NR==FNR { o[$2] = $1; next } { n[$2] = $1 }
    END {
      # ⚠️ test membership BEFORE reading o[k] — in awk merely reading o[k] creates the key
      for (k in n) { was = (k in o); d = n[k] - (was ? o[k] : 0); if (d >= 1e8 || d <= -1e8) printf "  %+8.2f GB  %s%s\n", d/1e9, k, was ? "" : "   (new, or was under 50 MB)" }
      for (k in o) if (!(k in n) && o[k] >= 1e8) printf "  %+8.2f GB  %s   (gone, or now under 50 MB)\n", -o[k]/1e9, k
    }' "$old" "$new" | sort -k1,1gr -k3,3
}

case "${1:-}" in
  disk-snapshot) disk_snapshot && echo "snapshot written: $D/disk/$(date -u +%F).tsv"; exit $? ;;
  disk-diff)     disk_diff "${2:-}"; exit $? ;;
esac

set -- $(grep "^cpu " /proc/stat); u1=$2;n1=$3;s1=$4;i1=$5;io1=${6:-0};q1=${7:-0};sq1=${8:-0}
idle1=$((i1+io1)); tot1=$((u1+n1+s1+i1+io1+q1+sq1))
sleep 1
set -- $(grep "^cpu " /proc/stat); u2=$2;n2=$3;s2=$4;i2=$5;io2=${6:-0};q2=${7:-0};sq2=${8:-0}
idle2=$((i2+io2)); tot2=$((u2+n2+s2+i2+io2+q2+sq2))
dt=$((tot2-tot1)); di=$((idle2-idle1)); cpu=0; [ "$dt" -gt 0 ] && cpu=$(( (100*(dt-di))/dt ))
mt=$(awk "/^MemTotal/{print \$2}" /proc/meminfo); ma=$(awk "/^MemAvailable/{print \$2}" /proc/meminfo)
mem=0; [ "${mt:-0}" -gt 0 ] && mem=$(( (100*(mt-ma))/mt ))
req=$(curl -s --max-time 2 http://127.0.0.1:8081/__nginx_status | awk "/server accepts/{getline; print \$3}")
[ -z "$req" ] && req=0
echo "$(date +%s),$cpu,$mem,$req" >> "$F"
tail -n 10100 "$F" > "$F.tmp" 2>/dev/null && mv "$F.tmp" "$F"
chmod 644 "$F"

# Daily disk snapshot — detached + locked so a slow du never delays or stacks up the per-minute sample.
# ⚠️ One ATTEMPT a day (.disk.tried), not "until the file exists": a du that keeps failing must not
#    re-run every minute. `disk-snapshot` retries by hand.
day=$(date -u +%F)
if [ ! -f "$D/disk/$day.tsv" ] && [ "$(cat "$D/.disk.tried" 2>/dev/null)" != "$day" ]; then
  ( flock -n 9 || exit 0; echo "$day" > "$D/.disk.tried"; disk_snapshot ) 9>"$D/.disk.lock" >/dev/null 2>&1 &
fi
