#!/usr/bin/env bash
# Woodpecker agent self-heal — prevents the daemon-timeout wedge cascade.
#
# CANONICAL HOME (Wave-2 P0.4 co-location): mgf-cloud-provision/ops/self-heal.sh.
# The copy in mgf-fed/deploy/woodpecker/ is DEPRECATED; the agent2 cloud-init
# (examples/woodpecker-agent-cloud-init.yml) embeds the same logic. Edit HERE.
#
# ROOT CAUSE (observed 2026-06-20): under high WOODPECKER_MAX_WORKFLOWS the agent
# fans out many step-containers at once; the Docker daemon times out on teardown
# (`context deadline exceeded` on docker.sock), so wp_* networks/volumes/
# containers leak. The next workflow then dies on `network ... already exists`,
# its workers never recover, and worker_count drifts to 0 (the "wedge").
#
# FIX: periodically reclaim leaked wp_* debris, and restart the agent if
# step-containers are stuck in "Created" (the wedge signal).
#
# ⚠️  THE CLAIM THAT USED TO BE HERE WAS FALSE, AND IT COST US RED PIPELINES.
# It read: "prune only removes resources NOT attached to a running container — the
# live pipeline is always safe." Both halves are true; the conclusion does not follow.
# BETWEEN two steps of a live pipeline, its network is attached to nothing, its scratch
# volume is dangling, and its just-finished step container has exited. "Unattached" is
# not "dead" — it is also what healthy work looks like for a few seconds, every few
# seconds. Every reaper here was racing live pipelines on a */5 tick.
# The age guard below (REAP_MIN_AGE_S) is what actually makes them safe.
#
# INSTALL (runbook §6.2): copy to the host, chmod 0750, root cron every 5 min:
#   server host:  */5 * * * * /opt/woodpecker/self-heal.sh
#   agent2 host:  */5 * * * * WOODPECKER_STACK_DIR=/opt/woodpecker-agent /opt/woodpecker-agent/self-heal.sh
set -uo pipefail
LOG=/var/log/woodpecker-self-heal.log
STACK_DIR="${WOODPECKER_STACK_DIR:-/opt/woodpecker}"
HEARTBEAT_URL_FILE="${HEARTBEAT_URL_FILE:-/etc/mgf/backup-heartbeat.url}"
HEARTBEAT_TOKEN_FILE="${HEARTBEAT_TOKEN_FILE:-/etc/mgf/backup-heartbeat.token}"
ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }

post_heartbeat() {
  # Success-only heartbeat to the OO backup_heartbeats stream (PAPER-02) — the
  # dead-man's-switch signal the estate watcher pages on. Best-effort emit: a
  # failed POST must never break the heal; the watcher pages on the MISSED
  # heartbeat either way. Same contract as backups/*: endpoint + auth ride in
  # the operator-pushed root-600 files /etc/mgf/backup-heartbeat.{url,token}.
  job="$1"
  if [ ! -r "$HEARTBEAT_URL_FILE" ] || [ ! -r "$HEARTBEAT_TOKEN_FILE" ]; then
    echo "$(ts) WARNING: heartbeat url/token file missing — no heartbeat sent (the watcher WILL page)" >> "$LOG"
    return 0
  fi
  payload=$(printf '[{"job":"%s","status":"ok","host":"%s","ts":"%s"}]' \
    "$job" "$(hostname)" "$(ts)")
  curl -fsS -m 20 -X POST -H 'Content-Type: application/json' \
    -H "Authorization: $(cat "$HEARTBEAT_TOKEN_FILE")" \
    -d "$payload" "$(cat "$HEARTBEAT_URL_FILE")" >/dev/null 2>&1 \
    || echo "$(ts) WARNING: heartbeat POST failed (the watcher WILL page)" >> "$LOG"
}

cd "$STACK_DIR" || exit 1

# THE AGE GUARD — the one property that makes every reaper below safe.
#
# "Not attached to a running container" is NOT the same as "dead", and believing it
# were is what made this script kill healthy pipelines. Between two steps of a LIVE
# pipeline, its network is attached to nothing, its scratch volume is dangling, and
# its just-finished step container is `exited`. Every reaper here was therefore racing
# live work on a */5 tick.
#
# PROVEN, not theorised: mgf-logship #95 (2026-07-12) — the `sbom` step started
# 06:29:59 and died 06:30:02 with `failed to set up container networking: network
# wp_<id>_0_default not found`, exit 126. 06:30:00 is a */5 tick. All 15 other steps
# passed. mgf-hrb-cap #3 and #5 died the same way the same day, on random victim steps.
#
# PAPER-09 already learned this for the *Created-container* reaper (§4 below) and added
# an age guard there. The lesson was never carried to the other three reapers five lines
# above it. So: ONE property, applied uniformly — only reap what is old enough to be
# certainly dead. A pipeline runs in minutes; leaked debris is hours old by definition.
REAP_MIN_AGE_S=${REAP_MIN_AGE_S:-3600}   # 1h. Nothing a live pipeline owns is this old.
now=$(date +%s)

older_than_guard() {  # $1 = docker object id; $2 = `container` | `volume`
  # (networks need no helper: `docker network prune --filter until=` does it natively)
  case "$2" in
    container) created=$(docker inspect -f '{{.Created}}'         "$1" 2>/dev/null) ;;
    volume)    created=$(docker volume inspect -f '{{.CreatedAt}}' "$1" 2>/dev/null) ;;
  esac
  [ -n "$created" ] || return 1
  created_s=$(date -d "$created" +%s 2>/dev/null) || return 1
  [ $(( now - created_s )) -ge "$REAP_MIN_AGE_S" ]
}

# 1. Reap dead wp_* containers — ONLY those older than the guard. An `exited` wp_*
#    container seconds old is the step that just finished in a pipeline still running.
dead=""
for id in $(docker ps -a --filter "name=wp_" --filter "status=exited" -q; \
            docker ps -a --filter "name=wp_" --filter "status=dead"   -q); do
  older_than_guard "$id" container && dead="$dead $id"
done
# shellcheck disable=SC2086
[ -n "$dead" ] && docker rm -f $dead >/dev/null 2>&1

# 2. Prune leaked networks — ONLY those older than the guard.
#    `docker network prune -f` on its own DELETES A LIVE PIPELINE'S NETWORK between
#    steps. `--filter until=` is the whole fix: it is a property, not a "remember to".
nets=$(docker network prune -f --filter "until=${REAP_MIN_AGE_S}s" 2>/dev/null | grep -c . || true)

# 3. Reap leaked wp_* scratch volumes — ONLY those older than the guard.
#    `dangling=true` matches a live pipeline's workspace volume between steps too, so
#    the age guard is load-bearing here for exactly the same reason. (The persistent
#    stack volumes are named `woodpecker_*`, not `wp_*`, so they are never matched.)
vols=""
for v in $(docker volume ls --filter name=wp_ --filter dangling=true -q); do
  older_than_guard "$v" volume && vols="$vols $v"
done
# shellcheck disable=SC2086
[ -n "$vols" ] && docker volume rm $vols >/dev/null 2>&1

# 3b. DISK-PRESSURE image reclaim (2026-07-14).
#
# THE GAP THIS CLOSES: the reapers above clean wp_* scratch (networks/volumes/containers),
# but nothing prunes leaked *images*. Old build layers accumulated to ~18 GB on agent2, and a
# release pipeline that installs a large dependency set (PySide6's Qt system libs) then hit
# `No space left on device` mid-`apt-get` — mgf-brand-qt's publish died on it. Constant image
# pruning is wrong (CI re-uses base images every run, so it would force re-pulls); so this only
# fires under real pressure, and only touches images UNUSED FOR >REAP_MIN_AGE_S. An image a
# live or recent pipeline uses is never removed.
disk_pct=$(df --output=pcent / | tail -1 | tr -dc '0-9')
img_reclaim=0
if [ "${disk_pct:-0}" -ge "${DISK_PRESSURE_PCT:-80}" ]; then
  img_reclaim=$(docker image prune -af --filter "until=${REAP_MIN_AGE_S}s" 2>/dev/null \
                | grep -c 'deleted\|untagged' || true)
fi

# 4. Wedge detection — WITH AN AGE GUARD (2026-07-11).
#
# THE BUG THIS FIXES: the old test was `created >= 1 && running == 0`, with no notion
# of HOW LONG a container had been in Created. But that is EXACTLY what a healthy
# pipeline looks like during its startup window — containers exist in `Created` and
# none is `Running` yet. When the */5 cron landed in that window it `docker rm -f`'d
# the containers that were about to run and restarted the agent, KILLING a healthy
# pipeline (it comes back as "Canceled" / every step failed at once). It had fired 11
# times. This is a CI-flakiness source, not only a safety net.
#
# A genuinely wedged container sits in `Created` indefinitely; a starting one flips to
# `Running` in seconds. So: only containers stuck in Created for >= STUCK_AGE_S count.
# The wedge recovery keeps working; the race disappears.
STUCK_AGE_S=${STUCK_AGE_S:-180}
DRY_RUN=${DRY_RUN:-0}

stuck_ids=""
for id in $(docker ps -a --filter "name=wp_" --filter "status=created" -q); do
  created_at=$(docker inspect -f '{{.Created}}' "$id" 2>/dev/null) || continue
  created_s=$(date -d "$created_at" +%s 2>/dev/null) || continue
  age=$(( now - created_s ))
  [ "$age" -ge "$STUCK_AGE_S" ] && stuck_ids="$stuck_ids $id"
done
running=$(docker ps --filter "name=wp_" --filter "status=running" -q | wc -l)
stuck=$(printf '%s' "$stuck_ids" | wc -w)

if [ "$stuck" -ge 1 ] && [ "$running" -eq 0 ]; then
  echo "$(ts) WEDGE: $stuck Created >= ${STUCK_AGE_S}s, 0 running -> restarting agent" >> "$LOG"
  if [ "$DRY_RUN" = "1" ]; then
    echo "$(ts)   DRY_RUN=1 — would rm$stuck_ids and restart the agent (no action taken)" >> "$LOG"
  else
    # shellcheck disable=SC2086
    docker rm -f $stuck_ids >/dev/null 2>&1
    docker compose restart woodpecker-agent >/dev/null 2>&1
  fi
fi

echo "$(ts) ok reaped_dead=$(printf '%s' "$dead" | wc -w) reaped_vols=$(printf '%s' "$vols" | wc -w) net_prune_lines=$nets img_reclaim=$img_reclaim disk=${disk_pct}% stuck=$stuck guard=${REAP_MIN_AGE_S}s" >> "$LOG"

post_heartbeat "self-heal"
