#!/usr/bin/env python3
"""Woodpecker agent reconciler (runs on the SERVER host via cron).

Recovers two failure modes that leave the fleet short of capacity and that neither
Docker's restart policy nor the `ping` healthcheck reliably handles:

  1. CRASH / DOWN — the agent container is not running. `docker kill` (and some real
     crashes) don't trip `restart: unless-stopped`; and a crash mid-pipeline leaves a
     stale network so a plain `docker compose restart` fails. Detected directly via the
     agent container's state — unambiguous (a crashed agent also leaks orphaned wp_*
     step containers, so a "0 build containers = idle" heuristic would wrongly read it
     as busy).
  2. STUCK — the container is UP + heartbeating + schedulable, but its server-side
     worker slots were orphaned (force-kill mid-pipeline, or a no_schedule toggle), so it
     sits idle while work pends. DELETE /api/agents can't clear it (409); only a restart.

Recovery (either case): reap leaked wp_* debris + prune networks + `docker compose up -d
--force-recreate` (a fresh container with fresh networking — handles the stale-network
crash case that `restart` can't). Local agent via docker; agent2 over SSH. A 2-consecutive
hit guard (tmpfs markers) avoids acting during a legitimate in-progress restart / the brief
between-pipeline window.

Auth (PAPER-01): the API token is MINTED PER-RUN from the Woodpecker DB — HS256 over
the user JWT with the user's DB `hash` as the key, byte-identical to Woodpecker's own
mint (and to the `mgf` workspace `tools/woodpecker_api.sh`, proven by
`rotate_woodpecker_token.sh --check`). A hash rotation therefore can never strand the
reconciler on a stale credential (the static /opt/woodpecker/reconciler.token did
exactly that, silently, 2026-06-23 → 2026-07-10). The token FILE is kept only as a
fallback for when the DB container itself is unreachable; each run logs which path
was used.

Liveness (PAPER-02): every SUCCESSFUL run POSTs a heartbeat to the OpenObserve
`backup_heartbeats` stream (same contract as backups/*: endpoint + auth ride in the
operator-pushed root-600 files /etc/mgf/backup-heartbeat.{url,token}); the estate
watcher pages when the heartbeat goes stale (ops/estate-watch-heartbeats.sh). The
heartbeat is fail-open — its failure must never break the heal.
"""

import base64
import hashlib
import hmac
import json
import os
import socket
import ssl
import subprocess
import time
import urllib.request

API = "https://195.15.241.48.sslip.io/api"
ADMIN_LOGIN = "magogi-admin"  # the admin user whose JWT is minted per-run
PG_CONTAINER = "woodpecker-woodpecker-postgres-1"
TOKEN_FILE = "/opt/woodpecker/reconciler.token"  # FALLBACK ONLY (DB unreachable)
KEY = "/opt/woodpecker/agent2_id"
LOG = "/var/log/woodpecker-reconciler.log"
MARK = "/run/wp-recon"
HEARTBEAT_URL_FILE = "/etc/mgf/backup-heartbeat.url"
HEARTBEAT_TOKEN_FILE = "/etc/mgf/backup-heartbeat.token"
HEARTBEAT_JOB = "agent-reconciler"
SSH = f"ssh -i {KEY} -o StrictHostKeyChecking=no -o ConnectTimeout=12 ubuntu@195.15.203.243"

# agent_id -> host (ids are stable; persisted in each agent's agent.conf)
AGENTS = {14: {"host": "local", "dir": "/opt/woodpecker"},
          15: {"host": "remote", "dir": "/opt/woodpecker-agent"}}
ctx = ssl.create_default_context()  # verified TLS — the API host serves a valid Let's Encrypt cert for its sslip.io name (SC: never disable cert verification)


def log(m):
    with open(LOG, "a") as f:
        f.write(time.strftime("%Y-%m-%dT%H:%M:%SZ ", time.gmtime()) + m + "\n")


# --------------------------------------------------------------------------
# Token mint (PAPER-01) — pure mint + DB fetch + fallback chain
# --------------------------------------------------------------------------
def mint_user_jwt(id_hash):
    """HS256-sign a Woodpecker user JWT from the DB's ``id|hash`` string.

    Pure (unit-testable without docker). Byte-identical to Woodpecker's own
    mint: header {"alg":"HS256","typ":"JWT"}, payload
    {"type":"user","user-id":"<id>"}, key = the user's DB `hash`, segments
    urlsafe-base64 without padding.
    """
    uid, uhash = id_hash.strip().split("|", 1)

    def b64(b):
        return base64.urlsafe_b64encode(b).rstrip(b"=")

    def seg(obj):
        return b64(json.dumps(obj, separators=(",", ":")).encode())

    msg = seg({"alg": "HS256", "typ": "JWT"}) + b"." + seg({"type": "user", "user-id": uid})
    return (msg + b"." + b64(hmac.new(uhash.encode(), msg, hashlib.sha256).digest())).decode()


def fetch_id_hash(login=ADMIN_LOGIN):
    """Read ``id|hash`` for *login* from the Woodpecker Postgres container.

    chr(124) is '|' — keeps the literal pipe out of the SQL/shell layers.
    Raises on any failure (non-zero exit, empty row) so the caller can fall back.
    """
    sql = f"select id||chr(124)||hash from users where login='{login}'"
    out = subprocess.run(
        ["sudo", "docker", "exec", PG_CONTAINER,
         "psql", "-U", "woodpecker", "-d", "woodpecker", "-tAc", sql],
        capture_output=True, text=True, timeout=20, check=True).stdout.strip()
    if "|" not in out:
        raise RuntimeError(f"no id|hash row for {login!r}")
    return out


def get_token():
    """Return ``(token, source)`` — mint from the DB, else the static file.

    The file path exists ONLY for the DB-unreachable case; the static token is
    what rotted silently for 17 days (PAPER-01), so the mint is always tried
    first.
    """
    try:
        return mint_user_jwt(fetch_id_hash()), "db-mint"
    except Exception as e:
        try:
            with open(TOKEN_FILE) as f:
                return f.read().strip(), f"file-fallback ({type(e).__name__} on mint)"
        except OSError:
            raise RuntimeError(
                f"token unavailable: mint failed ({type(e).__name__}) and no {TOKEN_FILE}"
            ) from e


# --------------------------------------------------------------------------
# Heartbeat (PAPER-02) — success-only, fail-open
# --------------------------------------------------------------------------
def heartbeat_request(job, url, token, host, ts, extra=None):
    """Build the heartbeat POST (pure): (url, body_bytes, headers).

    Record shape matches backups/*: job/status/host/ts (+ extras).
    """
    rec = {"job": job, "status": "ok", "host": host, "ts": ts}
    rec.update(extra or {})
    return url, json.dumps([rec]).encode(), {
        "Content-Type": "application/json", "Authorization": token}


def post_heartbeat(job=HEARTBEAT_JOB, extra=None):
    """POST a success heartbeat to the OO backup_heartbeats stream. FAIL-OPEN:
    any failure is logged and swallowed — a heartbeat problem must never break
    the heal; the watcher pages on the MISSED heartbeat either way."""
    try:
        with open(HEARTBEAT_URL_FILE) as f:
            url = f.read().strip()
        with open(HEARTBEAT_TOKEN_FILE) as f:
            token = f.read().strip()
    except OSError:
        log("heartbeat: url/token file missing — none sent (the watcher WILL page)")
        return False
    ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    url, body, headers = heartbeat_request(job, url, token, socket.gethostname(), ts, extra)
    try:
        urllib.request.urlopen(  # noqa: S310 — https URL from a root-600 operator file
            urllib.request.Request(url, data=body, headers=headers, method="POST"),
            timeout=20, context=ctx).close()
        return True
    except Exception as e:
        log(f"heartbeat POST failed: {type(e).__name__} (the watcher WILL page)")
        return False


# --------------------------------------------------------------------------
# Reconciler proper
# --------------------------------------------------------------------------
def api(path, token):
    req = urllib.request.Request(f"{API}/{path}", headers={"Authorization": f"Bearer {token}"})
    return json.load(urllib.request.urlopen(req, timeout=20, context=ctx))


def run(cmd, host, *, timeout=30):
    """Run a command; return its stdout on success, or None if it FAILED or the host was
    unreachable.

    None is NOT the same as empty output, and conflating them was a fail-open that defeated
    the dead-man's-switch: the old `run()` had no timeout and no exit-code check, so an SSH
    to an unreachable host returned `""`, `agent_up()` then evaluated `"" != "0"` -> True (an
    unreachable host read as UP), the reconciler took no action, and it STILL heartbeated
    "ok" — so the estate watcher never paged on a genuinely dead agent. Callers MUST treat
    None as "could not observe", never as a positive reading.
    """
    full = cmd if host == "local" else f"{SSH} {json.dumps(cmd)}"
    try:
        r = subprocess.run(full, shell=True, capture_output=True, text=True, timeout=timeout)
    except subprocess.TimeoutExpired:
        return None
    return r.stdout.strip() if r.returncode == 0 else None


def agent_up(info):
    """True only if we POSITIVELY observed a running agent container. Fails CLOSED: an
    unreachable host (`run` -> None) is not "up"."""
    pre = "" if info["host"] == "local" else "sudo "
    out = run(f"{pre}docker ps --filter name=woodpecker-agent --filter status=running -q | wc -l", info["host"])
    return out is not None and out not in ("", "0")


def build_containers(info):
    """How many build containers this agent has — in ANY state, not just Running.

    `docker ps -a`, deliberately (2026-07-11, the PAPER-09 lesson applied here).
    The old test counted only RUNNING containers, so an agent that is *starting* a
    pipeline — containers created, image still pulling — read as IDLE. With a job
    pending at the same time, that satisfies the "schedulable + idle while work
    pends" wedge signature, and a startup slower than the 2-hit guard (~3 min: a
    big image pull) would have got a perfectly healthy agent force-recreated,
    killing the very pipeline it was busy starting.

    A genuinely wedged agent (orphaned server-side slots) has NO wp_* containers at
    all; a starting one has them in `Created`. Counting every state is what actually
    tells those two apart.
    """
    pre = "" if info["host"] == "local" else "sudo "
    return run(f"{pre}docker ps -a --filter name=wp_ -q | wc -l", info["host"])


def recover(info):
    pre = "" if info["host"] == "local" else "sudo "
    cmd = (f"{pre}docker ps -aq --filter name=wp_ | xargs -r {pre}docker rm -f >/dev/null 2>&1; "
           f"{pre}docker network prune -f >/dev/null 2>&1; "
           f"cd {info['dir']} && {pre}docker compose up -d --force-recreate woodpecker-agent 2>&1 | tail -1")
    return run(cmd, info["host"])


def main():
    os.makedirs(MARK, exist_ok=True)
    try:
        token, token_src = get_token()
    except Exception as e:
        log(f"token error: {e}")
        return
    try:
        pending = api("queue/info", token).get("stats", {}).get("pending_count") or 0
        agents = {a["id"]: a for a in api("agents", token)}
    except Exception as e:
        log(f"api error: {type(e).__name__} (token={token_src})")
        return

    all_observed = True  # did we positively determine EVERY agent's state this run?
    for aid, info in AGENTS.items():
        marker = os.path.join(MARK, str(aid))
        a = agents.get(aid)
        schedulable = a is not None and not a.get("no_schedule")

        # Probe reachability directly so "unreachable" is distinct from "down". An
        # unreachable host cannot be reasoned about — we neither claim it healthy nor
        # silently skip it.
        pre = "" if info["host"] == "local" else "sudo "
        probe = run(f"{pre}docker ps --filter name=woodpecker-agent --filter status=running -q | wc -l", info["host"])
        reason = None
        if probe is None:
            all_observed = False
            reason = "host UNREACHABLE — cannot observe agent state"
        elif probe in ("", "0"):
            reason = "container DOWN (crashed)"
        elif schedulable and pending > 0 and build_containers(info) == "0":
            reason = f"schedulable + idle while {pending} pending (orphaned slots)"

        if reason:
            if os.path.exists(marker):            # 2nd consecutive => act
                log(f"RECOVER agent {aid} ({info['host']}): {reason}")
                log(f"  -> {recover(info)}")
                os.remove(marker)
            else:
                open(marker, "w").close()
        elif os.path.exists(marker):
            os.remove(marker)

    # The heartbeat is a dead-man's-switch: it must mean "I positively observed every agent
    # and all is well." If ANY host was unreachable we did NOT observe it, so we do NOT
    # heartbeat — the estate watcher then pages on the MISSED heartbeat, which is exactly what
    # a partially-blind reconciler should trigger. Heartbeating "ok" here was the fail-open.
    if all_observed:
        log(f"ok token={token_src} pending={pending}")
        post_heartbeat(extra={"token": token_src})
    else:
        log(f"DEGRADED token={token_src} pending={pending} — an agent host was unreachable; "
            f"NOT heartbeating (the watcher WILL page)")


if __name__ == "__main__":
    main()
