#!/usr/bin/env bash
# devpi-export-nightly.sh — nightly devpi state export (the fleet's sole package
# artery: every install + every release). Wave-2 P0.1 — backups as code.
#
#   stop devpi → devpi-export → start devpi → tar.gz → integrity + serving check
#   → prune (7 days) → optional object-storage upload → POST a heartbeat.
#
# INSTALL (devpi host, as root): copy to /usr/local/bin/devpi-export-nightly.sh
# (0750) + cron /etc/cron.d/mgf-backup-devpi — see
# examples/devpi-backup-cloud-init-fragment.yml. Heartbeat endpoint + token live
# in /etc/mgf/backup-heartbeat.{url,token} (pushed by the operator; NEVER
# committed). Restore: docs/backup-restore-runbook.md §2.
#
# Dead-man's-switch: the heartbeat is posted ONLY on success — any failure means
# a missed heartbeat, and the watcher pages on that. devpi is ALWAYS restarted
# (trap), even when the export fails or the script is killed mid-way.
set -euo pipefail
umask 077   # the export carries every private wheel + release history

BACKUP_DIR="${BACKUP_DIR:-/opt/backups}"
KEEP_DAYS="${KEEP_DAYS:-7}"
VENV="${VENV:-/opt/devpi/venv}"
SERVERDIR="${SERVERDIR:-/opt/devpi/server}"
DEVPI_USER="${DEVPI_USER:-devpi}"
EXPORT_DIR="${EXPORT_DIR:-/opt/devpi/exports}"
# post-backup serving check — localhost only (no committed endpoint value).
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:3141/root/pypi/+simple/}"
# Optional: an OpenStack Swift container (Infomaniak object storage). Empty = skip.
# Needs python-openstackclient + credentials (e.g. OS_CLOUD + a clouds.yaml under
# root's ~/.config/openstack/) on the host — values never committed.
SWIFT_CONTAINER="${SWIFT_CONTAINER:-}"
LOG="${LOG:-/opt/backups/backup.log}"   # local trail, uniform with config-snapshot-nightly.sh
HEARTBEAT_URL_FILE="${HEARTBEAT_URL_FILE:-/etc/mgf/backup-heartbeat.url}"
HEARTBEAT_TOKEN_FILE="${HEARTBEAT_TOKEN_FILE:-/etc/mgf/backup-heartbeat.token}"

post_heartbeat() {
  # Best-effort emit: a failed POST must not fail a succeeded backup — the
  # watcher pages on the MISSED heartbeat either way.
  job="$1"; artifact="$2"; bytes="$3"
  if [ ! -r "$HEARTBEAT_URL_FILE" ] || [ ! -r "$HEARTBEAT_TOKEN_FILE" ]; then
    echo "WARNING: heartbeat url/token file missing — no heartbeat sent (the watcher WILL page)" >&2
    return 0
  fi
  payload=$(printf '[{"job":"%s","status":"ok","host":"%s","artifact":"%s","bytes":%s,"ts":"%s"}]' \
    "$job" "$(hostname)" "$artifact" "${bytes:-0}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)")
  curl -fsS -m 20 -X POST -H 'Content-Type: application/json' \
    -H "Authorization: $(cat "$HEARTBEAT_TOKEN_FILE")" \
    -d "$payload" "$(cat "$HEARTBEAT_URL_FILE")" >/dev/null \
    || echo "WARNING: heartbeat POST failed (the watcher WILL page)" >&2
}

ts="$(date -u +%Y%m%d-%H%M%S)"
stage="$EXPORT_DIR/devpi-export-$ts"
out="$BACKUP_DIR/devpi-export-$ts.tar.gz"
mkdir -p "$BACKUP_DIR" "$EXPORT_DIR"

# The export needs the server stopped; the trap guarantees it comes back even on
# an unexpected exit (a SIGTERM/OOM between stop and start must NOT leave devpi down).
systemctl stop devpi
trap 'systemctl start devpi || true; rm -rf "$stage"' EXIT
# run the export as the devpi user (the serverdir is devpi-owned)
sudo -u "$DEVPI_USER" "$VENV/bin/devpi-export" --serverdir "$SERVERDIR" "$stage"
[ -f "$stage/dataindex.json" ] || { echo "ERROR: export has no dataindex.json" >&2; exit 1; }
systemctl start devpi
trap 'rm -rf "$stage"' EXIT   # devpi is back up; keep only the staging cleanup

tar czf "$out" -C "$EXPORT_DIR" "$(basename "$stage")"
bytes=$(stat -c %s "$out")
[ "$bytes" -gt 100000 ] || { echo "ERROR: export suspiciously small ($bytes bytes)" >&2; exit 1; }
gunzip -t "$out"   # integrity: a corrupt archive is a failed backup, not a heartbeat
echo "devpi export OK: $out ($bytes bytes)"

# prune local copies older than KEEP_DAYS
find "$BACKUP_DIR" -name 'devpi-export-*.tar.gz' -mtime "+$KEEP_DAYS" -delete

# the export is only worth a heartbeat if devpi is actually serving again
sleep 5
curl -fsS -m 20 -o /dev/null "$HEALTH_URL" \
  || { echo "ERROR: devpi not serving post-backup" >&2; exit 1; }

# optional off-host copy (Infomaniak object storage via Swift/OpenStack CLI)
if [ -n "$SWIFT_CONTAINER" ]; then
  openstack object create "$SWIFT_CONTAINER" "$out" --name "devpi/$(basename "$out")" >/dev/null
  echo "uploaded to swift container $SWIFT_CONTAINER"
fi

echo "[$ts] HEARTBEAT devpi-backup OK $(du -h "$out" | cut -f1)" >> "$LOG"
post_heartbeat "devpi-export" "$(basename "$out")" "$bytes"
