#!/usr/bin/env python3
"""Woodpecker agent reconciler (runs on the SERVER host via cron).

Recovers two failure modes that leave the fleet short of capacity and that neither
Docker's restart policy nor the `ping` healthcheck reliably handles:

  1. CRASH / DOWN — the agent container is not running. `docker kill` (and some real
     crashes) don't trip `restart: unless-stopped`; and a crash mid-pipeline leaves a
     stale network so a plain `docker compose restart` fails. Detected directly via the
     agent container's state — unambiguous (a crashed agent also leaks orphaned wp_*
     step containers, so a "0 build containers = idle" heuristic would wrongly read it
     as busy).
  2. STUCK — the container is UP + heartbeating + schedulable, but its server-side
     worker slots were orphaned (force-kill mid-pipeline, or a no_schedule toggle), so it
     sits idle while work pends. DELETE /api/agents can't clear it (409); only a restart.

Recovery (either case): reap leaked wp_* debris + prune networks + `docker compose up -d
--force-recreate` (a fresh container with fresh networking — handles the stale-network
crash case that `restart` can't). Local agent via docker; agent2 over SSH. A 2-consecutive
hit guard (tmpfs markers) avoids acting during a legitimate in-progress restart / the brief
between-pipeline window.
"""
import json, os, ssl, subprocess, time, urllib.request

API = "https://195.15.241.48.sslip.io/api"
TOKEN = open("/opt/woodpecker/reconciler.token").read().strip()
KEY = "/opt/woodpecker/agent2_id"
LOG = "/var/log/woodpecker-reconciler.log"
MARK = "/run/wp-recon"
SSH = f"ssh -i {KEY} -o StrictHostKeyChecking=no -o ConnectTimeout=12 ubuntu@195.15.203.243"

# agent_id -> host (ids are stable; persisted in each agent's agent.conf)
AGENTS = {14: {"host": "local", "dir": "/opt/woodpecker"},
          15: {"host": "remote", "dir": "/opt/woodpecker-agent"}}
ctx = ssl.create_default_context()  # verified TLS — the API host serves a valid Let's Encrypt cert for its sslip.io name (SC: never disable cert verification)
os.makedirs(MARK, exist_ok=True)

def log(m):
    with open(LOG, "a") as f:
        f.write(time.strftime("%Y-%m-%dT%H:%M:%SZ ", time.gmtime()) + m + "\n")

def api(path):
    req = urllib.request.Request(f"{API}/{path}", headers={"Authorization": f"Bearer {TOKEN}"})
    return json.load(urllib.request.urlopen(req, timeout=20, context=ctx))

def run(cmd, host):
    full = cmd if host == "local" else f"{SSH} {json.dumps(cmd)}"
    return subprocess.run(full, shell=True, capture_output=True, text=True).stdout.strip()

def agent_up(info):
    pre = "" if info["host"] == "local" else "sudo "
    return run(f"{pre}docker ps --filter name=woodpecker-agent --filter status=running -q | wc -l", info["host"]) != "0"

def build_containers(info):
    pre = "" if info["host"] == "local" else "sudo "
    return run(f"{pre}docker ps --filter name=wp_ -q | wc -l", info["host"])

def recover(info):
    pre = "" if info["host"] == "local" else "sudo "
    cmd = (f"{pre}docker ps -aq --filter name=wp_ | xargs -r {pre}docker rm -f >/dev/null 2>&1; "
           f"{pre}docker network prune -f >/dev/null 2>&1; "
           f"cd {info['dir']} && {pre}docker compose up -d --force-recreate woodpecker-agent 2>&1 | tail -1")
    return run(cmd, info["host"])

def main():
    try:
        pending = api("queue/info").get("stats", {}).get("pending_count") or 0
        agents = {a["id"]: a for a in api("agents")}
    except Exception as e:
        log(f"api error: {type(e).__name__}"); return

    for aid, info in AGENTS.items():
        marker = os.path.join(MARK, str(aid))
        a = agents.get(aid)
        schedulable = a is not None and not a.get("no_schedule")
        up = agent_up(info)
        reason = None
        if not up:
            reason = "container DOWN (crashed)"
        elif schedulable and pending > 0 and build_containers(info) == "0":
            reason = f"schedulable + idle while {pending} pending (orphaned slots)"
        if reason:
            if os.path.exists(marker):            # 2nd consecutive => act
                log(f"RECOVER agent {aid} ({info['host']}): {reason}")
                log(f"  -> {recover(info)}")
                os.remove(marker)
            else:
                open(marker, "w").close()
        elif os.path.exists(marker):
            os.remove(marker)

if __name__ == "__main__":
    main()
