#!/usr/bin/env bash
# Woodpecker agent self-heal — prevents the daemon-timeout wedge cascade.
#
# ROOT CAUSE (observed 2026-06-20): under high WOODPECKER_MAX_WORKFLOWS the agent
# fans out many step-containers at once; the Docker daemon times out on teardown
# (`context deadline exceeded` on docker.sock), so wp_* networks/volumes/
# containers leak. The next workflow then dies on `network ... already exists`,
# its workers never recover, and worker_count drifts to 0 (the "wedge").
#
# FIX: periodically reclaim leaked wp_* debris (prune only removes resources
# NOT attached to a running container — the live pipeline is always safe), and
# restart the agent if step-containers are stuck in "Created" (the wedge signal).
#
# INSTALL (see SETUP-WALKTHROUGH.md §5): copy to /opt/woodpecker/self-heal.sh,
# chmod 0750, and add a root cron — `*/5 * * * * /opt/woodpecker/self-heal.sh`.
set -uo pipefail
LOG=/var/log/woodpecker-self-heal.log
ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
cd /opt/woodpecker || exit 1

# 1. Reap dead wp_* containers (exited/created debris from failed teardowns).
dead=$(docker ps -a --filter "name=wp_" --filter "status=exited" -q; \
       docker ps -a --filter "name=wp_" --filter "status=dead"   -q)
[ -n "$dead" ] && echo "$dead" | xargs -r docker rm -f >/dev/null 2>&1

# 2. Prune unused networks + wp_* scratch volumes (safe: skips anything a running
#    container holds; the persistent stack volumes are named `woodpecker_*`, not
#    `wp_*`, so they are never matched). NOTE: plain `volume prune` only reaps
#    ANONYMOUS volumes — wp_* scratch volumes are NAMED, so target them explicitly.
nets=$(docker network prune -f 2>/dev/null | grep -c . || true)
docker volume ls --filter name=wp_ --filter dangling=true -q | xargs -r docker volume rm >/dev/null 2>&1

# 3. Wedge detection. A healthy pipeline's containers pass through "Created"
#    only transiently, so "Created" alone is not a wedge. Restart ONLY when
#    there is stuck Created debris AND nothing is actually Running — i.e. the
#    agent holds dead state but is doing no useful work. This can never kill a
#    live pipeline (a running pipeline => running>0 => no restart).
created=$(docker ps -a --filter "name=wp_" --filter "status=created" -q | wc -l)
running=$(docker ps   --filter "name=wp_" --filter "status=running" -q | wc -l)
stuck=0
if [ "$created" -ge 1 ] && [ "$running" -eq 0 ]; then
  stuck=$created
  echo "$(ts) WEDGE: $created stuck Created, 0 running -> restarting agent" >> "$LOG"
  docker ps -a --filter "name=wp_" --filter "status=created" -q | xargs -r docker rm -f >/dev/null 2>&1
  docker compose restart woodpecker-agent >/dev/null 2>&1
fi

echo "$(ts) ok reaped_dead=$([ -n "$dead" ] && echo "$dead" | wc -l || echo 0) net_prune_lines=$nets stuck=$stuck" >> "$LOG"
