#!/usr/bin/env bash
# Packet Station nightly backup on a native VM. Runs as root from packetstation-backup.timer,
# outside the app, so a broken app still backs up.
#
# For each configured stack it writes, into that stack's backup folder:
#   ps-<UTC stamp>.dump          pg_dump -Fc of the app database (custom format)
#   ps-<UTC stamp>.globals.sql   pg_dumpall --globals-only --no-role-passwords (roles, no password hashes)
#   last-success / last-failure  one line: "<ISO UTC time> <dump name>" (last-failure adds the reason)
#
# Each dump is written under a temporary name, checked with `pg_restore --list`, and only then
# renamed into place. The newest KEEP dumps (default 7) are kept, with their globals files. Any
# failure writes last-failure and the script exits non-zero after trying the remaining stacks.
#
# Usage:
#   ps-backup [--config FILE] [--keep N] [STACK ...]
#   ps-backup --stack NAME DATABASE FOLDER [--stack ...] [--keep N]
#
# Config file (default /etc/packetstation/backup.conf, or PS_BACKUP_CONFIG), one stack per line:
#   # name   database       folder
#   prod     packetstation  /var/backups/packetstation
# With STACK names given, only those stacks run. A VM has one stack; install.sh writes its line.
#
# pg_dump and pg_dumpall run as the postgres OS user (setpriv, no PAM) on the local socket, where
# PostgreSQL knows that user by peer authentication. No password is read, passed or printed. Their
# output goes to files root opened, so the postgres user needs no access to the folder.
# PG_BINDIR (default /usr/lib/postgresql/16/bin) picks the client tools, PG_AS the user switch.
#
# Globals: --no-role-passwords, so no password hash ever lands in a file (or in the PBS copy of
# it). The role passwords are in /etc/packetstation/secrets; deploy/vm/install.sh sets them.
#
# Files: dumps and globals are 0600. The markers are 0644 (a time and a file name, nothing else)
# so the worker, which reaches the folder through its group, can read them. When run as root the
# new files take the folder's owner and group.
#
# Not in the dump: the stored files (/var/lib/packetstation/assets), the env file and the secrets
# (/etc/packetstation). The PBS backup of the VM covers them. See docs/DEPLOY.md.

set -euo pipefail
umask 077

KEEP="${KEEP:-7}"
PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
# The command prefix that runs a client tool as the postgres OS user, split on spaces.
read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"

log() { printf '%s ps-backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }

usage() {
  sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' >&2
  exit 2
}

# ------------------------------------------------------------------ arguments

config="${PS_BACKUP_CONFIG:-/etc/packetstation/backup.conf}"
config_given=0
declare -a only=()
declare -a stacks=()   # each entry: "name|database|folder"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --config)
      [[ $# -ge 2 ]] || usage
      config="$2"; config_given=1; shift 2 ;;
    --stack)
      [[ $# -ge 4 ]] || usage
      stacks+=("$2|$3|$4"); shift 4 ;;
    --keep)
      [[ $# -ge 2 ]] || usage
      KEEP="$2"; shift 2 ;;
    -h|--help) usage ;;
    -*) log "unknown option $1"; usage ;;
    *) only+=("$1"); shift ;;
  esac
done

if ! [[ "$KEEP" =~ ^[1-9][0-9]*$ ]]; then
  log "KEEP must be a positive whole number"
  exit 2
fi

if [[ ${#stacks[@]} -eq 0 ]]; then
  if [[ ! -r "$config" ]]; then
    log "no --stack given and the config $config is missing or unreadable"
    exit 2
  fi
  while read -r name database folder extra; do
    [[ -z "${name:-}" || "${name:0:1}" == "#" ]] && continue
    if [[ -z "${folder:-}" || -n "${extra:-}" ]]; then
      log "config line for '$name' needs exactly: name database folder"
      exit 2
    fi
    stacks+=("$name|$database|$folder")
  done < "$config"
elif [[ $config_given -eq 1 ]]; then
  log "use --config or --stack, not both"
  exit 2
fi

if [[ ${#stacks[@]} -eq 0 ]]; then
  log "no stacks configured"
  exit 2
fi

# ------------------------------------------------------------------ helpers

# A PostgreSQL client tool: from PG_BINDIR when it is set, else from PATH.
pgbin() {
  if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
}

# Write a marker atomically: temp file in the same folder, then rename.
write_marker() {
  local folder="$1" marker="$2" text="$3" tmp
  tmp="$(mktemp "$folder/.${marker}.XXXXXX")"
  printf '%s\n' "$text" > "$tmp"
  chmod 0644 "$tmp"
  take_folder_owner "$folder" "$tmp"
  mv -f "$tmp" "$folder/$marker"
}

take_folder_owner() {
  local folder="$1"; shift
  if [[ "$(id -u)" == "0" ]]; then
    chown --reference="$folder" "$@" 2>/dev/null || true
  fi
}

# Keep the newest $KEEP dumps (names sort by their UTC stamp), and drop the globals of any removed dump.
rotate() {
  local folder="$1" f base
  local -a dumps=()
  while IFS= read -r f; do dumps+=("$f"); done < <(
    find "$folder" -maxdepth 1 -type f -name 'ps-*.dump' -printf '%f\n' | LC_ALL=C sort
  )
  local excess=$(( ${#dumps[@]} - KEEP ))
  local i
  for (( i = 0; i < excess; i++ )); do
    base="${dumps[$i]%.dump}"
    rm -f -- "$folder/${dumps[$i]}" "$folder/$base.globals.sql"
    log "removed old dump ${dumps[$i]}"
  done
  # Globals with no dump next to them (a run that failed after the globals step) go too.
  while IFS= read -r f; do
    base="${f%.globals.sql}"
    [[ -e "$folder/$base.dump" ]] || rm -f -- "$folder/$f"
  done < <(find "$folder" -maxdepth 1 -type f -name 'ps-*.globals.sql' -printf '%f\n')
}

# ------------------------------------------------------------------ one stack

# Set by backup_stack for the cleanup trap. Each stack runs in its own subshell (run_stack), so
# these never leak from one stack to the next.
cur_tmp_dump=""
cur_tmp_globals=""
cur_folder=""
cur_dump_name=""
failure_written=0

cleanup_tmp() {
  if [[ -n "$cur_tmp_dump" ]]; then rm -f -- "$cur_tmp_dump"; fi
  if [[ -n "$cur_tmp_globals" ]]; then rm -f -- "$cur_tmp_globals"; fi
  cur_tmp_dump=""
  cur_tmp_globals=""
}

# Record the failure and leave the stack's subshell.
fail_stack() {
  local reason="$1"
  cleanup_tmp
  log "FAILED: $reason"
  if [[ -n "$cur_folder" && -d "$cur_folder" ]]; then
    write_marker "$cur_folder" last-failure "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${cur_dump_name:--} $reason" || true
  fi
  failure_written=1
  exit 1
}

# Any command that fails without an explicit check still ends up as last-failure.
on_stack_exit() {
  local rc=$?
  cleanup_tmp
  if [[ $rc -ne 0 && $failure_written -eq 0 && -n "$cur_folder" && -d "$cur_folder" ]]; then
    log "FAILED: unexpected error (exit $rc)"
    write_marker "$cur_folder" last-failure \
      "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${cur_dump_name:--} unexpected error (exit $rc)" || true
  fi
}

backup_stack() {
  local name="$1" database="$2" folder="$3"
  local stamp globals_name started entries size

  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
  cur_dump_name="ps-$stamp.dump"
  globals_name="ps-$stamp.globals.sql"
  started=$SECONDS
  log "stack $name: database $database into $folder"

  if [[ ! -d "$folder" ]]; then
    # 0750: the worker reaches the folder through its group (docs/DEPLOY.md, "Backups on a VM").
    mkdir -p -- "$folder"
    chmod 0750 -- "$folder" 2>/dev/null || log "stack $name: could not set mode 0750 on $folder"
  fi
  cur_folder="$folder"
  [[ -w "$folder" ]] || fail_stack "$folder is not writable"
  [[ "$database" =~ ^[A-Za-z_][A-Za-z0-9_]{0,62}$ ]] || fail_stack "database name '$database' is not a plain name"

  if command -v flock >/dev/null 2>&1; then
    exec 9>"$folder/.ps-backup.lock"
    flock -n 9 || fail_stack "another backup of this stack is running"
  fi

  cur_tmp_dump="$(mktemp "$folder/.$cur_dump_name.XXXXXX.partial")"
  cur_tmp_globals="$(mktemp "$folder/.$globals_name.XXXXXX.partial")"

  "${PG_AS[@]}" "$(pgbin pg_dump)" -d "$database" -Fc > "$cur_tmp_dump" \
    || fail_stack "pg_dump failed"
  [[ -s "$cur_tmp_dump" ]] || fail_stack "pg_dump wrote nothing"

  "${PG_AS[@]}" "$(pgbin pg_dumpall)" --globals-only --no-role-passwords \
    > "$cur_tmp_globals" || fail_stack "pg_dumpall --globals-only failed"
  [[ -s "$cur_tmp_globals" ]] || fail_stack "pg_dumpall wrote nothing"

  # The check reads the dump from stdin and needs no database. Entry lines start with the
  # entry's id; comment lines start with a semicolon.
  entries="$("$(pgbin pg_restore)" --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
    || fail_stack "pg_restore --list could not read the dump"
  [[ "$entries" -ge 1 ]] || fail_stack "pg_restore --list found no entries"

  chmod 0600 "$cur_tmp_dump" "$cur_tmp_globals"
  take_folder_owner "$folder" "$cur_tmp_dump" "$cur_tmp_globals"
  mv -f -- "$cur_tmp_globals" "$folder/$globals_name"
  cur_tmp_globals=""
  mv -f -- "$cur_tmp_dump" "$folder/$cur_dump_name"
  cur_tmp_dump=""

  size="$(stat -c %s "$folder/$cur_dump_name")"
  write_marker "$folder" last-success "$(date -u +%Y-%m-%dT%H:%M:%SZ) $cur_dump_name"
  rotate "$folder"
  log "stack $name: ok, $cur_dump_name, $size bytes, $entries entries, $(( SECONDS - started )) s"
}

# One stack in its own subshell with errexit on, so a failure ends that stack only.
run_stack() (
  set -euo pipefail
  trap on_stack_exit EXIT
  backup_stack "$@"
)

# ------------------------------------------------------------------ main

failures=0
ran=0
for entry in "${stacks[@]}"; do
  IFS='|' read -r s_name s_database s_folder <<< "$entry"
  if [[ ${#only[@]} -gt 0 ]]; then
    wanted=0
    for o in "${only[@]}"; do
      if [[ "$o" == "$s_name" ]]; then wanted=1; fi
    done
    [[ $wanted -eq 1 ]] || continue
  fi
  ran=$(( ran + 1 ))
  # Not in an if or ||: that would switch errexit off inside the subshell.
  set +e
  run_stack "$s_name" "$s_database" "$s_folder"
  rc=$?
  set -e
  if [[ $rc -ne 0 ]]; then
    failures=$(( failures + 1 ))
  fi
done

if [[ $ran -eq 0 ]]; then
  log "no configured stack matched: ${only[*]}"
  exit 2
fi
if [[ $failures -gt 0 ]]; then
  log "$failures of $ran stacks failed"
  exit 1
fi
log "all $ran stacks backed up"