#!/usr/bin/env bash
# Off-host backup mirror — pull last night's artifacts from all three backup hosts
# so that losing a single VM cannot destroy the estate's memory.
#
# Runs on agent2 (deliberately a DIFFERENT host from every store it mirrors), 03:00,
# after devpi 02:17 / pg 02:20 / OO 02:30. Reads as `root@host` over a dedicated key.
#
# 2026-07-11 HARDENING — the mirror had a hole that hid a real disaster-recovery
# failure for a week. It validated that an artifact EXISTED (present, >10 KB, gzip-
# valid) but never that it was FRESH. So when devpi's backup cron turned out to have
# never been installed at all (mgf-cloud-provision PAPER-04), this script faithfully
# re-mirrored the same 2026-07-04 export every night and logged `backup-mirror OK`
# — for seven consecutive nights. A stale-but-valid backup passed the watchdog.
#
# Three changes, all aimed at that failure mode:
#   1. FRESHNESS (the fix): each source artifact must be newer than MAX_AGE_H (26 h
#      by default — one nightly cycle plus slack). A stale artifact is now a FAILURE
#      and pages, because an upstream backup that stopped being produced is exactly
#      the thing this mirror exists to notice.
#   2. HEARTBEAT: success POSTs to the OO `backup_heartbeats` stream (job=backup-mirror)
#      so that a dead MIRROR CRON is itself caught — previously the mirror could only
#      report its own failures; if the cron stopped, the silence looked like health.
#      Fail-open: a heartbeat problem must never break the mirror.
#   3. PAGE guard: `PAGE=0` suppresses issue creation, so the failure path can be
#      fire-tested without filing a false alarm at the founder.
set -uo pipefail

TS=$(date +%Y%m%d-%H%M)
LOG=${LOG:-/opt/watcher/pull.log}
MIR=${MIR:-/opt/backup-mirror}
MAX_AGE_H=${MAX_AGE_H:-26}          # an artifact older than this = FAILURE (see #1)
PAGE=${PAGE:-1}                     # 0 = do not file a Codeberg issue (fire-testing)
KEEP=${KEEP:-7}                     # copies retained per source
HEARTBEAT_URL_FILE="${HEARTBEAT_URL_FILE:-/etc/mgf/backup-heartbeat.url}"
HEARTBEAT_TOKEN_FILE="${HEARTBEAT_TOKEN_FILE:-/etc/mgf/backup-heartbeat.token}"

K="-i ${PULL_KEY:-/opt/watcher/backup-pull-key} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30"
# PAPER-08: the data restores were PROVEN by the 2026-07-11 drill — these cfg-* sources
# are the CONFIG + CREDENTIALS a service needs to actually START (OO root creds,
# WOODPECKER_AGENT_SECRET, the collector token, the pager token, and the mirror's own
# pull key). Without them a host loss is recoverable on paper only.
declare -A SRC=( [devpi]="195.15.203.250:/opt/backups/devpi-export-*.tar.gz" \
                 [pg]="195.15.241.48:/opt/backups/woodpecker-pg-*.sql.gz" \
                 [oo]="195.15.203.114:/opt/backups/oo-data-*.tar.gz" \
                 [cfg-wp]="195.15.241.48:/opt/backups/config-woodpecker-prod-*.tar.gz" \
                 [cfg-devpi]="195.15.203.250:/opt/backups/config-devpi-prod-*.tar.gz" \
                 [cfg-oo]="195.15.203.114:/opt/backups/config-openobserve-prod-*.tar.gz" )

post_heartbeat() {
  # Success-only dead-man's-switch (job=backup-mirror). FAIL-OPEN by construction.
  [ -r "$HEARTBEAT_URL_FILE" ] && [ -r "$HEARTBEAT_TOKEN_FILE" ] || {
    echo "[$TS] WARNING: heartbeat url/token missing — none sent (watcher WILL page)" >> "$LOG"
    return 0
  }
  curl -fsS -m 20 -X POST -H 'Content-Type: application/json' \
    -H "Authorization: $(cat "$HEARTBEAT_TOKEN_FILE")" \
    -d "[{\"job\":\"backup-mirror\",\"status\":\"ok\",\"host\":\"$(hostname)\",\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}]" \
    "$(cat "$HEARTBEAT_URL_FILE")" >/dev/null 2>&1 \
    || echo "[$TS] WARNING: heartbeat POST failed (watcher WILL page)" >> "$LOG"
}

FAILED=""
for name in devpi pg oo cfg-wp cfg-devpi cfg-oo; do
  host=${SRC[$name]%%:*}; glob=${SRC[$name]#*:}
  mkdir -p "$MIR/$name"

  # Newest artifact upstream, plus its age in seconds — asked for in ONE round trip so
  # the freshness verdict is about the file we are actually mirroring.
  read -r latest age_s < <(ssh $K "root@$host" \
    "f=\$(ls -1t $glob 2>/dev/null | head -1); [ -n \"\$f\" ] && echo \"\$f \$(( \$(date +%s) - \$(stat -c %Y \"\$f\") ))\"" 2>/dev/null)

  if [ -z "${latest:-}" ]; then FAILED="$FAILED $name(no-artifact)"; continue; fi

  # (1) FRESHNESS — the check whose absence hid PAPER-04 for a week.
  if [ "${age_s:-999999}" -gt $(( MAX_AGE_H * 3600 )) ]; then
    FAILED="$FAILED $name(STALE:$(( age_s / 3600 ))h>${MAX_AGE_H}h)"
    # still mirror it (a stale backup beats no backup) — but the run FAILS and pages.
  fi

  base=$(basename "$latest")
  if [ ! -f "$MIR/$name/$base" ]; then
    scp -q $K "root@$host:$latest" "$MIR/$name/$base.part" && mv "$MIR/$name/$base.part" "$MIR/$name/$base" \
      || { FAILED="$FAILED $name(copy)"; continue; }
  fi
  # Minimum plausible size is PER-SOURCE: the data stores are multi-MB, but a config
  # snapshot is legitimately ~1-4 KB (a tar of a handful of .env/credential files).
  # A single 10 KB floor would red the mirror every night on a perfectly good artifact.
  case "$name" in cfg-*) MIN=200 ;; *) MIN=10000 ;; esac
  SZ=$(stat -c%s "$MIR/$name/$base" 2>/dev/null || echo 0)
  [ "$SZ" -gt "$MIN" ] || FAILED="$FAILED $name(too-small:$SZ<$MIN)"
  gunzip -t "$MIR/$name/$base" 2>/dev/null || FAILED="$FAILED $name(gzip)"
  ls -1t "$MIR/$name"/* 2>/dev/null | tail -n +$(( KEEP + 1 )) | xargs -r rm -f
done

if [ -n "$FAILED" ]; then
  echo "[$TS] HEARTBEAT backup-mirror FAILED:$FAILED" >> "$LOG"
  if [ "$PAGE" = "1" ]; then
    TOK=$(cat /opt/watcher/.codeberg-token 2>/dev/null)
    if [ -n "$TOK" ]; then
      TITLE="[estate] backup mirror failure:$FAILED"
      EXIST=$(curl -sf -m 20 -H "Authorization: token $TOK" \
        "https://codeberg.org/api/v1/repos/magogi-admin/mgf-standard/issues?state=open&type=issues&q=backup+mirror+failure" \
        | grep -c '"title"' || true)
      [ "$EXIST" = "0" ] && curl -sf -m 20 -X POST -H "Authorization: token $TOK" -H "Content-Type: application/json" \
        -d "{\"title\":\"$TITLE\",\"body\":\"pull-backups.sh on agent2 failed at $TS:$FAILED. A STALE:* entry means an upstream backup job has stopped producing artifacts — check that host's cron (PAPER-04 precedent: devpi's cron had never been installed). See /opt/watcher/pull.log\"}" \
        "https://codeberg.org/api/v1/repos/magogi-admin/mgf-standard/issues" >/dev/null
    fi
  fi
  exit 1
fi

echo "[$TS] HEARTBEAT backup-mirror OK $(du -sh "$MIR" | cut -f1)" >> "$LOG"
post_heartbeat
