#!/usr/bin/env bash
# oo-snapshot-nightly.sh — nightly snapshot of the OpenObserve data volume (the
# central log store, incl. the backup-heartbeat stream itself). Wave-2 P0.1 —
# backups as code.
#
#   cold-stop → tar the docker volume → restart → integrity + serving check →
#   prune (7 days) → POST a heartbeat.
#
# INSTALL (openobserve host, as root): copy to
# /usr/local/bin/oo-snapshot-nightly.sh (0750) + cron /etc/cron.d/mgf-backup-oo
# — see examples/openobserve-backup-cloud-init-fragment.yml. Heartbeat endpoint
# + token in /etc/mgf/backup-heartbeat.{url,token} (operator-pushed; NEVER
# committed). Restore: docs/backup-restore-runbook.md §4.
#
# A COLD snapshot (OO briefly stopped) is the default: it guarantees a consistent
# tar of the file-backed store, and logship's fail-open spool covers the brief
# outage (RPO 24h). Set LIVE_SNAPSHOT=1 to tar without stopping (no downtime, at
# the risk of a torn snapshot).
#
# Dead-man's-switch: heartbeat only on success; the watcher pages on a miss. The
# trap guarantees OO restarts even if the tar fails or the script is killed.
set -euo pipefail
umask 077   # the log store carries operational telemetry — never world-readable

BACKUP_DIR="${BACKUP_DIR:-/opt/backups}"
KEEP_DAYS="${KEEP_DAYS:-7}"
VOLUME="${VOLUME:-openobserve_openobserve-data}"
CONTAINER="${CONTAINER:-openobserve-openobserve-1}"
LIVE_SNAPSHOT="${LIVE_SNAPSHOT:-0}"
# post-backup serving check — localhost only (no committed endpoint value).
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:5080/}"
LOG="${LOG:-/opt/backups/backup.log}"   # local trail, uniform with config-snapshot-nightly.sh
HEARTBEAT_URL_FILE="${HEARTBEAT_URL_FILE:-/etc/mgf/backup-heartbeat.url}"
HEARTBEAT_TOKEN_FILE="${HEARTBEAT_TOKEN_FILE:-/etc/mgf/backup-heartbeat.token}"

post_heartbeat() {
  # Best-effort emit: a failed POST must not fail a succeeded backup — the
  # watcher pages on the MISSED heartbeat either way.
  job="$1"; artifact="$2"; bytes="$3"
  if [ ! -r "$HEARTBEAT_URL_FILE" ] || [ ! -r "$HEARTBEAT_TOKEN_FILE" ]; then
    echo "WARNING: heartbeat url/token file missing — no heartbeat sent (the watcher WILL page)" >&2
    return 0
  fi
  payload=$(printf '[{"job":"%s","status":"ok","host":"%s","artifact":"%s","bytes":%s,"ts":"%s"}]' \
    "$job" "$(hostname)" "$artifact" "${bytes:-0}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)")
  curl -fsS -m 20 -X POST -H 'Content-Type: application/json' \
    -H "Authorization: $(cat "$HEARTBEAT_TOKEN_FILE")" \
    -d "$payload" "$(cat "$HEARTBEAT_URL_FILE")" >/dev/null \
    || echo "WARNING: heartbeat POST failed (the watcher WILL page)" >&2
}

ts="$(date -u +%Y%m%d-%H%M%S)"
out="$BACKUP_DIR/oo-data-$ts.tar.gz"
mkdir -p "$BACKUP_DIR"

mountpoint="$(docker volume inspect -f '{{.Mountpoint}}' "$VOLUME")"
[ -d "$mountpoint" ] || { echo "ERROR: volume mountpoint not found: $VOLUME" >&2; exit 1; }

if [ "$LIVE_SNAPSHOT" = "1" ]; then
  # live tar: tolerate 'file changed as we read it' (tar rc 1); rc >1 is a real error
  rc=0; tar --warning=no-file-changed -czf "$out" -C "$mountpoint" . || rc=$?
  [ "$rc" -le 1 ] || { echo "ERROR: tar failed (rc=$rc)" >&2; exit "$rc"; }
else
  # cold snapshot — stop OO, tar, and guarantee restart whatever happens
  docker stop "$CONTAINER" >/dev/null
  trap 'docker start "$CONTAINER" >/dev/null 2>&1 || true' EXIT
  tar -czf "$out" -C "$mountpoint" .
  docker start "$CONTAINER" >/dev/null
  trap - EXIT
fi

bytes=$(stat -c %s "$out")
[ "$bytes" -gt 10000 ] || { echo "ERROR: snapshot suspiciously small ($bytes bytes)" >&2; exit 1; }
gunzip -t "$out"   # integrity: a corrupt archive is a failed backup, not a heartbeat
echo "openobserve snapshot OK: $out ($bytes bytes)"

find "$BACKUP_DIR" -name 'oo-data-*.tar.gz' -mtime "+$KEEP_DAYS" -delete

# the snapshot is only worth a heartbeat if OO is actually serving again
sleep 8
curl -fsS -m 20 -o /dev/null "$HEALTH_URL" \
  || { echo "ERROR: OpenObserve not serving post-backup" >&2; exit 1; }

echo "[$ts] HEARTBEAT oo-backup OK $(du -h "$out" | cut -f1)" >> "$LOG"
post_heartbeat "oo-snapshot" "$(basename "$out")" "$bytes"
