#!/usr/bin/env bash
# estate-watch-heartbeats.sh — heartbeat FRESHNESS check for the estate watcher
# (PAPER-02 / MGF_INFRA §5 "heartbeat freshness" planned scope).
#
# WHAT: queries the OpenObserve `backup_heartbeats` stream and reports every
# monitored job whose latest success-heartbeat is older than its window. This is
# the dead-man's-switch READ side: the self-heal crons (agent-reconciler.py,
# self-heal.sh) and the nightly backups (backups/*) each POST a success-only
# heartbeat; a job that stops running — or starts failing silently, like the
# reconciler's 17-day 401 outage (PAPER-01) — simply stops heartbeating, and
# this check pages on the staleness. No new secret distribution: it reuses the
# same operator-pushed root-600 files /etc/mgf/backup-heartbeat.{url,token}
# (the OO credential is valid for _search as well as ingest).
#
# WINDOWS: self-heal jobs run */3 and */5 → stale > 30 min pages (N≈30 min per
# PAPER-02); nightly backups → stale > 26 h pages. Override via CHECKS.
#
# DEPLOY (agent2, 195.15.203.243 — deliberately a different host from every
# store it watches):
#   1. copy to /opt/watcher/estate-watch-heartbeats.sh, root:root 0750
#   2. push /etc/mgf/backup-heartbeat.{url,token} (root-600) to agent2 if not
#      already present (they exist on the backup hosts; agent2 may lack them)
#   3. wire into /opt/watcher/estate-watch.sh (the live */10 cron): run this
#      script, and on non-zero exit route its stdout through the SAME deduped
#      Codeberg-issue path the uptime probes use, e.g.:
#        if ! out=$(/opt/watcher/estate-watch-heartbeats.sh); then
#          file_issue "estate-watch: stale heartbeats" "$out"
#        fi
#      (file_issue = the existing dedup + issue-create helper in estate-watch.sh)
#
# EXIT: 0 all fresh; 1 at least one job stale; 2 the check itself could not run
# (missing cred files / OO unreachable / bad response) — page on that too: a
# blind watcher is a dead watcher.
#
# NOTE: windows are per-JOB. `self-heal` runs on BOTH hosts under one job name,
# so one host's dead cron hides behind the other's heartbeats; scope per-host
# checks by extending CHECKS with the `host` field once hostnames are pinned
# (records carry "host": $(hostname)).
set -uo pipefail

HEARTBEAT_URL_FILE="${HEARTBEAT_URL_FILE:-/etc/mgf/backup-heartbeat.url}"
HEARTBEAT_TOKEN_FILE="${HEARTBEAT_TOKEN_FILE:-/etc/mgf/backup-heartbeat.token}"
# job:max_age_seconds — 1800 = 30 min (self-heal layer), 93600 = 26 h (backups)
# The DEFAULT must list every job that heartbeats, or a MANUAL run silently reports
# on a narrower set than the live watcher and reads as "all clear" (found 2026-07-11:
# the default lagged the watcher by two jobs). The live watcher passes CHECKS explicitly.
CHECKS="${CHECKS:-agent-reconciler:1800 self-heal:1800 pg-dump:93600 devpi-export:93600 oo-snapshot:93600 backup-mirror:93600 config-snapshot:93600}"

if [ ! -r "$HEARTBEAT_URL_FILE" ] || [ ! -r "$HEARTBEAT_TOKEN_FILE" ]; then
  echo "CHECK-FAILED: heartbeat url/token file missing on this host (push /etc/mgf/backup-heartbeat.{url,token})"
  exit 2
fi

ingest_url="$(cat "$HEARTBEAT_URL_FILE")"
auth="$(cat "$HEARTBEAT_TOKEN_FILE")"
# ingest URL is .../api/<org>/backup_heartbeats/_json → search is .../api/<org>/_search
search_url="${ingest_url%/*/_json}/_search"
stream="$(basename "${ingest_url%/_json}")"
if [ "$search_url" = "$ingest_url" ]; then
  echo "CHECK-FAILED: cannot derive _search endpoint from $HEARTBEAT_URL_FILE (expected .../<stream>/_json)"
  exit 2
fi

now_us=$(( $(date -u +%s) * 1000000 ))
stale=0 failed=0

for spec in $CHECKS; do
  job="${spec%%:*}"; window="${spec##*:}"
  start_us=$(( now_us - window * 1000000 ))
  body=$(printf '{"query":{"sql":"SELECT count(*) AS n FROM \\"%s\\" WHERE job = '"'"'%s'"'"' AND status = '"'"'ok'"'"'","start_time":%s,"end_time":%s}}' \
    "$stream" "$job" "$start_us" "$now_us")
  resp=$(curl -fsS -m 20 -X POST -H 'Content-Type: application/json' \
    -H "Authorization: $auth" -d "$body" "$search_url" 2>/dev/null)
  if [ -z "$resp" ]; then
    echo "CHECK-FAILED: OO _search unreachable/refused for job=$job"
    failed=1
    continue
  fi
  n=$(printf '%s' "$resp" | python3 -c \
    'import sys,json;h=json.load(sys.stdin).get("hits",[]);print(h[0].get("n",0) if h else 0)' \
    2>/dev/null) || { echo "CHECK-FAILED: unparseable _search response for job=$job"; failed=1; continue; }
  if [ "${n:-0}" -eq 0 ]; then
    echo "STALE: job=$job — no success heartbeat in the last ${window}s"
    stale=1
  fi
done

[ "$failed" -eq 1 ] && exit 2
[ "$stale" -eq 1 ] && exit 1
echo "heartbeats OK ($CHECKS)"
exit 0
