Native VM install kit: systemd units, launcher, install and upgrade scripts; backups, restore and doctor without docker exec; DEPLOY.md VM, NPM and NFS sections

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 18:08:48 -06:00
parent f025479358
commit 45a1d1a42a
22 changed files with 2357 additions and 413 deletions
@@ -0,0 +1,510 @@
 #!/usr/bin/env bash
 # Packet Station native VM install. Run by Spencer as root on a new test or production VM, from an
 # unpacked release (make-release.sh). Idempotent: run it again after a change and it only fixes
 # what differs. It never overwrites the env file or a secret, never starts the app or the worker,
 # and never touches VISUAL, M-Files or AD.
 #
 # Usage:
 #   install.sh --stack test|prod [--ca-bundle FILE] [--generate-secrets] [--skip-packages]
 #   install.sh --files-only
 #
 #   --stack S            test or prod. Picks the env file example. Read from the env file when it exists.
 #   --ca-bundle FILE     the PESCO CA chain (PEM, scripts/Export-PescoCaBundle.ps1). Installed as
 #                        /etc/packetstation/ca/pesco-ca.pem and into the system trust store.
 #   --generate-secrets   create the four internal secrets that are missing (both PostgreSQL role
 #                        passwords, session_key, seal_key) from /dev/urandom. Never replaces a file.
 #   --skip-packages      leave apt alone (repositories and packages already in place).
 #   --files-only         only the systemd units, the ps-* commands, backup.conf and the firewall
 #                        table. ps-upgrade runs this from the new release.
 #
 # What it does, in order:
 #   1. apt: PostgreSQL 16 (PGDG), ODBC Driver 18 (Microsoft), Python 3.12, nftables, nfs-common.
 #   2. User and group packetstation, uid and gid 10001 (the image's uid, and the NFS share's).
 #   3. Folders: /etc/packetstation, /var/lib/packetstation/assets, /var/backups/packetstation,
 #      /opt/packetstation, the archive mount point /mnt/packetstation-archive.
 #   4. /etc/packetstation/packetstation.env from the stack's example, if missing.
 #   5. The PESCO CA, when --ca-bundle is given.
 #   6. Secret files: owner root:packetstation, mode 0440, one line, no carriage return.
 #   7. PostgreSQL: socket only (conf.d/packetstation.conf), scram for ps_app and ps_migrate on the
 #      socket (pg_hba.conf), the roles and the database as deploy/postgres/initdb makes them.
 #   8. Units, commands, backup timer, firewall (the files-only part).
 set -euo pipefail
 umask 022
 PS_USER=packetstation
 PS_ID=10001
 PG_VERSION=16
 PG_CLUSTER=main
 ETC=/etc/packetstation
 SECRETS="$ETC/secrets"
 ENV_FILE="$ETC/packetstation.env"
 CA_FILE="$ETC/ca/pesco-ca.pem"
 NFT_FILE="$ETC/packetstation.nft"
 BACKUP_CONF="$ETC/backup.conf"
 ASSETS=/var/lib/packetstation/assets
 BACKUPS=/var/backups/packetstation
 ARCHIVE_MOUNT=/mnt/packetstation-archive
 OPT=/opt/packetstation
 SBIN=/usr/local/sbin
 UNIT_DIR=/etc/systemd/system
 PG_ETC="/etc/postgresql/$PG_VERSION/$PG_CLUSTER"
 PG_UNIT="postgresql@$PG_VERSION-$PG_CLUSTER.service"
 PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin"
 DB_LOCALE=en_US.UTF-8
 # Signing keys, checked by fingerprint before apt trusts them.
 PGDG_KEY_URL=https://www.postgresql.org/media/keys/ACCC4CF8.asc
 PGDG_KEY_FPR=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
 MS_KEY_URL=https://packages.microsoft.com/keys/microsoft.asc
 MS_KEY_FPR=BC528686B50D79E339D3721CEB3E94ADBE1229CF
 ALL_SECRETS=(postgres_migrate_password postgres_app_password visualkit_password mfileskit_password
             ldap_bind_password session_key seal_key)
 GENERATED_SECRETS=(postgres_migrate_password postgres_app_password session_key seal_key)
 here="$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd -P)"
 release="$(cd -- "$here/../.." && pwd -P)"
 log() { printf 'install: %s\n' "$*" >&2; }
 die() { printf 'install: ERROR: %s\n' "$*" >&2; exit 1; }
 usage() { sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
 stack=""
 ca_bundle=""
 generate=0
 skip_packages=0
 files_only=0
 while [[ $# -gt 0 ]]; do
  case "$1" in
    --stack) [[ $# -ge 2 ]] || usage; stack="$2"; shift 2 ;;
    --ca-bundle) [[ $# -ge 2 ]] || usage; ca_bundle="$2"; shift 2 ;;
    --generate-secrets) generate=1; shift ;;
    --skip-packages) skip_packages=1; shift ;;
    --files-only) files_only=1; shift ;;
    -h|--help) usage ;;
    *) log "unknown argument $1"; usage ;;
  esac
 done
 [[ "$(id -u)" == "0" ]] || die "run as root"
 [[ -z "$stack" || "$stack" == "test" || "$stack" == "prod" ]] || die "--stack must be test or prod"
 # ------------------------------------------------------------------ helpers
 # A KEY=value from the env file, with surrounding quotes removed. Empty when unset.
 env_value() {
  local key="$1" line value
  [[ -r "$ENV_FILE" ]] || return 0
  line="$(grep -E "^[[:space:]]*$key=" "$ENV_FILE" | tail -n 1 || true)"
  value="${line#*=}"
  value="${value%$'\r'}"
  value="${value#\"}"; value="${value%\"}"
  value="${value#\'}"; value="${value%\'}"
  printf '%s' "$value"
 }
 # Install a text file only when it differs. Strips carriage returns (a Windows checkout).
 # Prints "changed" when it wrote the file.
 put_file() {
  local src="$1" dst="$2" owner="$3" mode="$4" tmp
  tmp="$(mktemp)"
  sed 's/\r$//' -- "$src" > "$tmp"
  if [[ -f "$dst" ]] && cmp -s -- "$tmp" "$dst"; then
    chown "$owner" -- "$dst"; chmod "$mode" -- "$dst"
    rm -f -- "$tmp"
    return 0
  fi
  install -o "${owner%%:*}" -g "${owner##*:}" -m "$mode" -- "$tmp" "$dst"
  rm -f -- "$tmp"
  echo changed
 }
 as_postgres() {
  setpriv --reuid=postgres --regid=postgres --init-groups -- "$@"
 }
 is_ipv4() { [[ "$1" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; }
 is_ipv6() { [[ "$1" =~ ^[0-9A-Fa-f:]+$ && "$1" == *:* ]]; }
 # ------------------------------------------------------------------ 1. packages
 os_id=""
 os_version=""
 os_codename=""
 read_os() {
  # shellcheck disable=SC1091
  os_id="$(. /etc/os-release && printf '%s' "${ID:-}")"
  # shellcheck disable=SC1091
  os_version="$(. /etc/os-release && printf '%s' "${VERSION_ID:-}")"
  # shellcheck disable=SC1091
  os_codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")"
 }
 fetch_key() {
  local url="$1" fpr="$2" dst="$3" tmp found
  tmp="$(mktemp -d)"
  curl -fsSL --proto '=https' "$url" -o "$tmp/key.asc"
  found="$(gpg --batch --show-keys --with-colons "$tmp/key.asc" 2>/dev/null | awk -F: '$1 == "fpr" && !seen { print $10; seen = 1 }')"
  if [[ "$found" != "$fpr" ]]; then
    rm -rf -- "$tmp"
    die "the key from $url has fingerprint '${found:-none}', expected $fpr. Check it before going on."
  fi
  gpg --batch --yes --dearmor -o "$tmp/key.gpg" "$tmp/key.asc"
  install -o root -g root -m 0644 "$tmp/key.gpg" "$dst"
  rm -rf -- "$tmp"
 }
 step_packages() {
  local ms_repo python_pkgs=()
  case "$os_id $os_version" in
    "ubuntu 24.04")
      ms_repo="https://packages.microsoft.com/ubuntu/24.04/prod $os_codename main"
      python_pkgs=(python3.12 python3.12-venv) ;;
    "debian 12")
      ms_repo="https://packages.microsoft.com/debian/12/prod $os_codename main"
      log "Debian 12 ships Python 3.11. Packet Station needs 3.12: give ps-upgrade PYTHON=/path/to/python3.12." ;;
    *) die "unsupported OS '$os_id $os_version'. Supported: Ubuntu 24.04, Debian 12." ;;
  esac
  export DEBIAN_FRONTEND=noninteractive
  log "apt: base packages"
  apt-get update -q
  apt-get install -y -q --no-install-recommends \
    ca-certificates curl gnupg locales tzdata nftables nfs-common util-linux coreutils findutils
  # The database locale. Generated before PostgreSQL so its first cluster knows it too.
  if ! locale -a 2>/dev/null | grep -qiE '^en_US\.utf-?8$'; then
    log "generating locale $DB_LOCALE"
    if [[ -f /etc/locale.gen ]]; then
      sed -i 's/^# *\(en_US.UTF-8 UTF-8\)/\1/' /etc/locale.gen
    fi
    locale-gen en_US.UTF-8 >/dev/null
  fi
  log "apt: PGDG and Microsoft repositories"
  fetch_key "$PGDG_KEY_URL" "$PGDG_KEY_FPR" /usr/share/keyrings/pgdg.gpg
  fetch_key "$MS_KEY_URL" "$MS_KEY_FPR" /usr/share/keyrings/microsoft-prod.gpg
  printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/pgdg.gpg] https://apt.postgresql.org/pub/repos/apt %s-pgdg main\n' \
    "$os_codename" > /etc/apt/sources.list.d/pgdg.list
  printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] %s\n' "$ms_repo" \
    > /etc/apt/sources.list.d/microsoft-prod.list
  apt-get update -q
  log "apt: PostgreSQL $PG_VERSION, ODBC Driver 18, Python"
  ACCEPT_EULA=Y apt-get install -y -q --no-install-recommends \
    "postgresql-$PG_VERSION" "postgresql-client-$PG_VERSION" msodbcsql18 unixodbc "${python_pkgs[@]}"
  odbcinst -q -d -n "ODBC Driver 18 for SQL Server" >/dev/null \
    || die "ODBC Driver 18 for SQL Server is not registered with unixODBC"
 }
 # ------------------------------------------------------------------ 2. user
 step_user() {
  local g u
  g="$(getent group "$PS_USER" | cut -d: -f3 || true)"
  if [[ -z "$g" ]]; then
    if getent group "$PS_ID" >/dev/null; then
      die "gid $PS_ID is taken by group $(getent group "$PS_ID" | cut -d: -f1)"
    fi
    groupadd --system --gid "$PS_ID" "$PS_USER"
    log "created group $PS_USER ($PS_ID)"
  elif [[ "$g" != "$PS_ID" ]]; then
    die "group $PS_USER has gid $g, expected $PS_ID (the NFS share and the backups expect $PS_ID)"
  fi
  u="$(id -u "$PS_USER" 2>/dev/null || true)"
  if [[ -z "$u" ]]; then
    if getent passwd "$PS_ID" >/dev/null; then
      die "uid $PS_ID is taken by user $(getent passwd "$PS_ID" | cut -d: -f1)"
    fi
    useradd --system --uid "$PS_ID" --gid "$PS_ID" --home-dir /var/lib/packetstation --no-create-home \
      --shell /usr/sbin/nologin --comment "Packet Station" "$PS_USER"
    log "created user $PS_USER ($PS_ID)"
  elif [[ "$u" != "$PS_ID" ]]; then
    die "user $PS_USER has uid $u, expected $PS_ID"
  fi
 }
 # ------------------------------------------------------------------ 3. folders
 step_dirs() {
  install -d -o root -g "$PS_USER" -m 0750 "$ETC" "$SECRETS"
  install -d -o root -g root -m 0755 "$ETC/ca" "$OPT"
  install -d -o root -g "$PS_USER" -m 0750 /var/lib/packetstation "$BACKUPS"
  install -d -o "$PS_USER" -g "$PS_USER" -m 0750 "$ASSETS"
  # The mount point stays root's and read-only to packetstation, so while the share is not mounted
  # nothing can be written into the VM's own disk by mistake.
  if mountpoint -q "$ARCHIVE_MOUNT" 2>/dev/null; then
    log "$ARCHIVE_MOUNT is mounted; left as it is"
  else
    install -d -o root -g root -m 0755 "$ARCHIVE_MOUNT"
  fi
 }
 # ------------------------------------------------------------------ 4. env file
 step_env() {
  local example have
  if [[ -e "$ENV_FILE" ]]; then
    have="$(env_value STACK)"
    if [[ -n "$stack" && "$have" != "$stack" ]]; then
      die "$ENV_FILE says STACK=$have, but --stack $stack was given"
    fi
    chown root:"$PS_USER" "$ENV_FILE"; chmod 0640 "$ENV_FILE"
    log "kept $ENV_FILE (STACK=$have)"
    return 0
  fi
  [[ -n "$stack" ]] || die "no $ENV_FILE yet: give --stack test or --stack prod"
  example="$here/$stack.env.example"
  [[ -r "$example" ]] || die "missing $example"
  put_file "$example" "$ENV_FILE" "root:$PS_USER" 0640 >/dev/null
  log "wrote $ENV_FILE from $(basename -- "$example"). Read it through before the first start."
 }
 # ------------------------------------------------------------------ 5. CA
 step_ca() {
  if [[ -z "$ca_bundle" ]]; then
    [[ -f "$CA_FILE" ]] || log "no PESCO CA yet: run again with --ca-bundle FILE (LDAP and SMTP need it)"
    return 0
  fi
  [[ -r "$ca_bundle" ]] || die "cannot read $ca_bundle"
  grep -q 'BEGIN CERTIFICATE' "$ca_bundle" || die "$ca_bundle holds no PEM certificate"
  if grep -q 'PRIVATE KEY' "$ca_bundle"; then
    die "$ca_bundle holds a private key. Export only the certificates."
  fi
  put_file "$ca_bundle" "$CA_FILE" root:root 0644 >/dev/null
  # update-ca-certificates wants one certificate per .crt file.
  local dir=/usr/local/share/ca-certificates/pesco tmp
  tmp="$(mktemp -d)"
  sed 's/\r$//' -- "$CA_FILE" | awk -v d="$tmp" '
    /-----BEGIN CERTIFICATE-----/ { n++; f = sprintf("%s/pesco-ca-%02d.crt", d, n) }
    f { print > f }
    /-----END CERTIFICATE-----/ { close(f); f = "" }'
  rm -rf -- "$dir"
  install -d -o root -g root -m 0755 "$dir"
  install -o root -g root -m 0644 "$tmp"/*.crt "$dir"/
  rm -rf -- "$tmp"
  update-ca-certificates >/dev/null
  log "PESCO CA installed: $CA_FILE and the system trust store ($(find "$dir" -name '*.crt' | wc -l) certificates)"
 }
 # ------------------------------------------------------------------ 6. secrets
 new_secret() {
  # 32 random bytes as 64 hex characters. Held in a variable and written by a builtin, never on argv.
  local name="$1" value tmp
  value="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')"
  [[ ${#value} -eq 64 ]] || die "could not read /dev/urandom"
  tmp="$(mktemp "$SECRETS/.$name.XXXXXX")"
  printf '%s' "$value" > "$tmp"
  unset value
  chown root:"$PS_USER" "$tmp"; chmod 0440 "$tmp"
  mv -- "$tmp" "$SECRETS/$name"
  log "generated secret $name"
 }
 missing_secrets=()
 step_secrets() {
  local name f g
  missing_secrets=()
  for name in "${ALL_SECRETS[@]}"; do
    f="$SECRETS/$name"
    if [[ ! -e "$f" && $generate -eq 1 ]]; then
      for g in "${GENERATED_SECRETS[@]}"; do
        if [[ "$g" == "$name" ]]; then new_secret "$name"; fi
      done
    fi
    if [[ ! -e "$f" ]]; then
      missing_secrets+=("$name")
      continue
    fi
    chown root:"$PS_USER" -- "$f"; chmod 0440 -- "$f"
    [[ -s "$f" ]] || die "secret $name is empty"
    if [[ "$(wc -l < "$f")" -gt 1 ]] || grep -q $'\r' -- "$f"; then
      die "secret $name must be one line with no carriage return"
    fi
  done
  if [[ ${#missing_secrets[@]} -gt 0 ]]; then
    log "secrets still missing in $SECRETS: ${missing_secrets[*]}"
  fi
 }
 have_secret() { [[ -s "$SECRETS/$1" ]]; }
 # ------------------------------------------------------------------ 7. PostgreSQL
 step_postgres() {
  local changed="" hba="$PG_ETC/pg_hba.conf" tmp
  [[ -d "$PG_ETC" ]] || die "no PostgreSQL cluster $PG_VERSION/$PG_CLUSTER in $PG_ETC"
  grep -Eq "^[[:space:]]*include_dir[[:space:]]*=[[:space:]]*'conf\.d'" "$PG_ETC/postgresql.conf" \
    || die "$PG_ETC/postgresql.conf has no include_dir = 'conf.d'"
  install -d -o postgres -g postgres -m 0755 "$PG_ETC/conf.d"
  changed="$(put_file "$here/postgres/packetstation.conf" "$PG_ETC/conf.d/packetstation.conf" postgres:postgres 0644)"
  # ps_app and ps_migrate log in on the socket with their passwords (scram). The block goes first,
  # before Debian's "local all all peer", which would refuse them (the OS user is packetstation).
  if ! grep -q '^# packetstation begin' "$hba"; then
    tmp="$(mktemp)"
    awk '
      !done && $0 !~ /^[[:space:]]*(#|$)/ {
        print "# packetstation begin (deploy/vm/install.sh)"
        print "local   packetstation   ps_app,ps_migrate   scram-sha-256"
        print "# packetstation end"
        print ""
        done = 1
      }
      { print }' "$hba" > "$tmp"
    install -o postgres -g postgres -m 0640 "$tmp" "$hba"
    rm -f -- "$tmp"
    log "pg_hba.conf: ps_app and ps_migrate use scram-sha-256 on the socket"
    [[ -n "$changed" ]] || changed=reload
  fi
  systemctl enable --quiet "$PG_UNIT" 2>/dev/null || true
  if [[ "$changed" == "changed" ]]; then
    log "restarting PostgreSQL (conf.d/packetstation.conf changed)"
    systemctl restart "$PG_UNIT"
  elif [[ "$changed" == "reload" ]]; then
    systemctl start "$PG_UNIT"
    systemctl reload "$PG_UNIT"
  else
    systemctl start "$PG_UNIT"
  fi
  if ! have_secret postgres_migrate_password || ! have_secret postgres_app_password; then
    log "skipping roles and database: the PostgreSQL secrets are missing (or use --generate-secrets)"
    return 0
  fi
  log "PostgreSQL roles ps_migrate and ps_app, database packetstation"
  # The passwords reach psql through its environment (\getenv), never through argv or a file.
  # A failed statement is not logged with its text (log_min_error_statement), so a password
  # cannot land in the server log.
  PS_MIGRATE_PW="$(<"$SECRETS/postgres_migrate_password")"
  PS_APP_PW="$(<"$SECRETS/postgres_app_password")"
  export PS_MIGRATE_PW PS_APP_PW
  as_postgres "$PG_BIN/psql" -X -q -v ON_ERROR_STOP=1 -d postgres <<SQL
 SET log_min_error_statement = panic;
 SET log_statement = 'none';
 \\getenv migrate_pw PS_MIGRATE_PW
 \\getenv app_pw PS_APP_PW
 SELECT 'CREATE ROLE ps_migrate' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_migrate') \\gexec
 SELECT 'CREATE ROLE ps_app' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_app') \\gexec
 ALTER ROLE ps_migrate WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'migrate_pw';
 ALTER ROLE ps_app     WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'app_pw';
 \\unset migrate_pw
 \\unset app_pw
 SELECT format('CREATE DATABASE packetstation OWNER ps_migrate ENCODING %L LOCALE %L TEMPLATE template0',
              'UTF8', '$DB_LOCALE')
 WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'packetstation') \\gexec
 REVOKE ALL ON DATABASE packetstation FROM PUBLIC;
 GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app;
 \\connect packetstation
 REVOKE ALL ON SCHEMA public FROM PUBLIC;
 ALTER SCHEMA public OWNER TO ps_migrate;
 SQL
  unset PS_MIGRATE_PW PS_APP_PW
 }
 # ------------------------------------------------------------------ 8. files
 write_firewall() {
  local proxy rule tmp
  proxy="$(env_value TRUSTED_PROXY_HOST)"
  if is_ipv4 "$proxy"; then
    rule="ip saddr $proxy"
  elif is_ipv6 "$proxy"; then
    rule="ip6 saddr $proxy"
  else
    die "TRUSTED_PROXY_HOST in $ENV_FILE must be NPM's IP address (it is '${proxy}')"
  fi
  tmp="$(mktemp)"
  cat > "$tmp" <<NFT
 #!/usr/sbin/nft -f
 # Written by deploy/vm/install.sh from TRUSTED_PROXY_HOST in $ENV_FILE. Do not edit:
 # change the env file and run install.sh --files-only. Loaded by packetstation-firewall.service.
 # Port 8000 (the app) answers only NPM on docker-1 and the VM itself. Nothing else is filtered here.
 table inet packetstation
 delete table inet packetstation
 table inet packetstation {
  chain input {
    type filter hook input priority filter; policy accept;
    tcp dport 8000 iifname "lo" accept
    tcp dport 8000 $rule accept
    tcp dport 8000 counter drop
  }
 }
 NFT
  if command -v nft >/dev/null 2>&1; then
    nft -c -f "$tmp" || { rm -f -- "$tmp"; die "nft refused the generated table"; }
  fi
  put_file "$tmp" "$NFT_FILE" root:root 0644 >/dev/null
  rm -f -- "$tmp"
 }
 step_files() {
  local f st
  [[ -r "$ENV_FILE" ]] || die "no $ENV_FILE: run install.sh --stack test|prod first"
  st="$(env_value STACK)"
  [[ "$st" == "test" || "$st" == "prod" ]] || die "STACK in $ENV_FILE must be test or prod"
  for f in "$here"/systemd/*.service; do
    put_file "$f" "$UNIT_DIR/$(basename -- "$f")" root:root 0644 >/dev/null
  done
  put_file "$release/scripts/backup/packetstation-backup.service" "$UNIT_DIR/packetstation-backup.service" root:root 0644 >/dev/null
  put_file "$release/scripts/backup/packetstation-backup.timer" "$UNIT_DIR/packetstation-backup.timer" root:root 0644 >/dev/null
  put_file "$here/ps-run.sh" "$SBIN/ps-run" root:root 0755 >/dev/null
  put_file "$here/ps-doctor.sh" "$SBIN/ps-doctor" root:root 0755 >/dev/null
  put_file "$here/upgrade.sh" "$SBIN/ps-upgrade" root:root 0755 >/dev/null
  put_file "$release/scripts/backup/ps-backup.sh" "$SBIN/ps-backup" root:root 0755 >/dev/null
  put_file "$release/scripts/backup/ps-restore.sh" "$SBIN/ps-restore" root:root 0755 >/dev/null
  if [[ ! -e "$BACKUP_CONF" ]]; then
    printf '# name  database       folder\n%s  packetstation  %s\n' "$st" "$BACKUPS" > "$BACKUP_CONF"
    chmod 0644 "$BACKUP_CONF"
    log "wrote $BACKUP_CONF"
  fi
  write_firewall
  systemctl daemon-reload
  systemctl enable --quiet packetstation-firewall.service packetstation-backup.timer
  if systemctl is-active --quiet packetstation-firewall.service; then
    systemctl reload packetstation-firewall.service
  else
    systemctl start packetstation-firewall.service
  fi
  systemctl start packetstation-backup.timer
  log "units, ps-run, ps-doctor, ps-upgrade, ps-backup, ps-restore and the firewall table are in place"
 }
 # ------------------------------------------------------------------ main
 if [[ $files_only -eq 1 ]]; then
  step_files
  exit 0
 fi
 read_os
 if [[ $skip_packages -eq 0 ]]; then
  step_packages
 fi
 for tool in setpriv systemctl install mountpoint; do
  command -v "$tool" >/dev/null 2>&1 || die "$tool is not installed"
 done
 step_user
 step_dirs
 step_env
 step_ca
 step_secrets
 step_postgres
 step_files
 cat >&2 <<EOF
 install: done for STACK=$(env_value STACK).
 install: next:
  - read $ENV_FILE through
  - secrets still missing: ${missing_secrets[*]:-none}
  - install a release:  ps-upgrade /path/to/packetstation-<build>.tar.gz
  - the app and the worker are not enabled or started. Starting them is a separate step (docs/DEPLOY.md).
 EOF
@@ -0,0 +1,107 @@
 #!/usr/bin/env bash
 # Build a release archive for the native VMs. Runs on a build host as a normal user, inside a git
 # checkout of this repository: docker-1 (with --image) or a PC with Node (with --npm). Never on a VM.
 #
 # Usage:
 #   make-release.sh --image gt.pescoinc.biz/spencerm/packetstation:sha-<sha> [--out DIR]
 #   make-release.sh --npm [--ref REF] [--build NAME] [--out DIR]
 #
 #   --image IMAGE   take the built frontend out of a CI-tested image (docker create and docker cp,
 #                   nothing runs). The build name and the commit come from the image's build label,
 #                   so the source and the frontend are the tested commit.
 #   --npm           build the frontend here with npm ci and npm run build, from the same commit.
 #   --ref REF       the commit for --npm (default HEAD). Uncommitted changes are never included.
 #   --build NAME    the build name for --npm (default sha-<full sha>, as CI names images).
 #   --out DIR       where to write the archive (default the current folder).
 #
 # Writes packetstation-<build>.tar.gz and packetstation-<build>.tar.gz.sha256. The archive holds one
 # folder, packetstation-<build>/, with BUILD, pyproject.toml, README.md, server/, deploy/, scripts/,
 # docs/ and static/ (the built frontend). Python packages are not in it: ps-upgrade builds the venv
 # on the VM from the pinned requirements.
 set -euo pipefail
 umask 022
 DOCKER="${DOCKER:-docker}"
 LABEL=biz.pescoinc.packetstation.build
 log() { printf 'make-release: %s\n' "$*" >&2; }
 die() { printf 'make-release: ERROR: %s\n' "$*" >&2; exit 1; }
 usage() { sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
 image=""
 npm=0
 ref=""
 build=""
 out="."
 while [[ $# -gt 0 ]]; do
  case "$1" in
    --image) [[ $# -ge 2 ]] || usage; image="$2"; shift 2 ;;
    --npm) npm=1; shift ;;
    --ref) [[ $# -ge 2 ]] || usage; ref="$2"; shift 2 ;;
    --build) [[ $# -ge 2 ]] || usage; build="$2"; shift 2 ;;
    --out) [[ $# -ge 2 ]] || usage; out="$2"; shift 2 ;;
    -h|--help) usage ;;
    *) log "unknown argument $1"; usage ;;
  esac
 done
 if [[ -n "$image" && $npm -eq 1 ]] || [[ -z "$image" && $npm -eq 0 ]]; then
  die "give exactly one of --image or --npm"
 fi
 if [[ -n "$image" && ( -n "$ref" || -n "$build" ) ]]; then
  die "--ref and --build come from the image's label with --image"
 fi
 [[ "$image" != *:latest && ( -z "$image" || "$image" == *:* ) ]] || die "pull an image by its tag, never latest"
 repo="$(git rev-parse --show-toplevel 2>/dev/null)" || die "run inside a git checkout of Packet Station"
 cd -- "$repo"
 if [[ -n "$image" ]]; then
  "$DOCKER" image inspect "$image" >/dev/null 2>&1 || "$DOCKER" pull "$image" >/dev/null
  build="$("$DOCKER" image inspect -f "{{ index .Config.Labels \"$LABEL\" }}" "$image")"
  [[ "$build" =~ ^sha-[0-9a-f]{7,40}$ ]] || die "image label $LABEL is '$build', expected sha-<commit>"
  ref="${build#sha-}"
 else
  ref="${ref:-HEAD}"
 fi
 sha="$(git rev-parse --verify --quiet "$ref^{commit}")" || die "commit $ref is not in this checkout (git fetch first)"
 if [[ -z "$build" ]]; then
  build="sha-$sha"
 fi
 [[ "$build" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] || die "build name '$build' must be letters, digits, '.', '_' or '-'"
 name="packetstation-$build"
 mkdir -p -- "$out"
 out="$(cd -- "$out" && pwd)"
 work="$(mktemp -d)"
 cid=""
 cleanup() {
  if [[ -n "$cid" ]]; then "$DOCKER" rm -f "$cid" >/dev/null 2>&1 || true; fi
  rm -rf -- "$work"
 }
 trap cleanup EXIT
 log "source of $sha"
 # LF line endings whatever the checkout's autocrlf: these files run on Linux.
 git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix="$name/" "$sha" \
  pyproject.toml README.md server deploy scripts docs | tar -xf - -C "$work"
 printf '%s\n' "$build" > "$work/$name/BUILD"
 [[ -f "$work/$name/deploy/vm/ps-launch.sh" ]] || die "commit $sha has no deploy/vm (older than phase 10)"
 chmod 0755 "$work/$name"/deploy/vm/*.sh "$work/$name"/scripts/backup/*.sh
 if [[ -n "$image" ]]; then
  log "frontend from $image"
  cid="$("$DOCKER" create "$image")"
  "$DOCKER" cp "$cid:/app/static" "$work/$name/static"
 else
  log "frontend: npm ci and npm run build at $sha"
  git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix=frontend/ "$sha:frontend" \
    | tar -xf - -C "$work"
  ( cd -- "$work/frontend" && npm ci --no-audit --no-fund && npm run build ) >&2
  cp -R -- "$work/frontend/dist" "$work/$name/static"
 fi
 [[ -f "$work/$name/static/index.html" ]] || die "no static/index.html in the built frontend"
 log "writing $out/$name.tar.gz"
 tar -czf "$out/$name.tar.gz" -C "$work" --owner=0 --group=0 --numeric-owner --sort=name "$name"
 ( cd -- "$out" && sha256sum -- "$name.tar.gz" > "$name.tar.gz.sha256" )
 log "done: $out/$name.tar.gz ($(wc -c < "$out/$name.tar.gz") bytes) and its .sha256"
@@ -0,0 +1,13 @@
 # Packet Station on this VM. Installed by deploy/vm/install.sh as
 # /etc/postgresql/16/main/conf.d/packetstation.conf. A change here needs a PostgreSQL restart.
 # Local socket only. The app, the worker, migrate and the backups all connect through
 # /var/run/postgresql. Nothing on the network can reach PostgreSQL.
 listen_addresses = ''
 # The same as the postgres container on docker-1: the server clock zone is UTC.
 # The app shows times in SITE_TZ.
 timezone = 'UTC'
 log_timezone = 'UTC'
 password_encryption = 'scram-sha-256'
@@ -0,0 +1,102 @@
 # Packet Station production settings on the production VM (native, no Docker).
 # deploy/vm/install.sh --stack prod copies this to /etc/packetstation/packetstation.env
 # (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
 # It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
 # Starting packetstation-worker with this file polls VISUAL production on its schedule.
 # Spencer's first start of the worker is the approval.
 #
 # systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
 # Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
 # Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
 # ---- Stack and site ----
 # prod, test, ci or dev. Guards against crossing stacks.
 STACK=prod
 # The VISUAL site in every call. The docker-1 test stack runs with PESCO.
 SITE_ID=PESCO
 # Site time zone for schedules and display. The processes run in UTC.
 SITE_TZ=America/Denver
 # Public URL, used in links and mail.
 PUBLIC_URL=https://packetstation.pescoinc.biz
 # NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
 # opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
 TRUSTED_PROXY_HOST=192.168.101.12
 # DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
 LOG_LEVEL=INFO
 # ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
 # A folder means the Unix socket. PostgreSQL listens on no TCP port.
 PG_HOST=/var/run/postgresql
 PG_PORT=5432
 PG_DATABASE=packetstation
 PG_APP_USER=ps_app
 PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
 PG_MIGRATE_USER=ps_migrate
 PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
 # ---- Keys and folders ----
 # The services get the secrets as systemd credentials and ps-launch.sh points these at them.
 # The paths here are the files those credentials are loaded from.
 SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
 SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
 # Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
 ASSETS_DIR=/var/lib/packetstation/assets
 # Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
 BACKUP_DIR=/var/backups/packetstation
 # The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
 # `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
 # Admin on the Admin page, File locations.
 #ARCHIVE_ROOT=/mnt/packetstation-archive
 # ---- Uploads and storage ----
 UPLOAD_MAX_BYTES=26214400
 UPLOAD_MAX_BYTES_BOM=5242880
 ASSETS_QUOTA_WARN_PCT=85
 # 0 measures the file system under ASSETS_DIR.
 ASSETS_QUOTA_BYTES=0
 # ---- ClamAV (optional, off: Gitea issue 19) ----
 # Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
 #CLAMAV_HOST=127.0.0.1
 CLAMAV_PORT=3310
 CLAMAV_TIMEOUT_SECONDS=30
 # ---- LDAP (PESCO Active Directory) ----
 LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
 LDAP_BASE_DN=DC=PESCO,DC=com
 LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
 LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
 LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
 LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
 # The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
 LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
 LDAP_STARTTLS=false
 LDAP_ALLOW_INSECURE=false
 # ---- Mail (relay app-03, no auth) ----
 SMTP_HOST=app-03.pesco.com
 SMTP_PORT=25
 SMTP_FROM=[email protected]
 SMTP_STARTTLS=true
 SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
 # Production sends mail.
 NOTIFY_DRY_RUN=false
 # ---- VisualKit (VISUAL ERP, read only) ----
 # Production, set explicitly. Never on the test VM.
 VISUAL_SERVER=vsql.pesco.com
 VISUAL_DATABASE=PESCOINC
 VISUAL_USER=visualkit_ro
 VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
 # no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
 # store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
 VISUAL_ENCRYPT=no
 VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
 # ---- MFilesKit (M-Files, read only) ----
 MFILES_URL=https://mfiles.pescoinc.biz/REST
 MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
 MFILES_USERNAME=mfileskitRO
 # ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
 MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
 # Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.
@@ -0,0 +1,7 @@
 #!/usr/bin/env bash
 # `packetstation doctor` on this VM, run the way the services run (see ps-run.sh).
 # Installed as /usr/local/sbin/ps-doctor. Run as root: sudo ps-doctor [--skip visual,mfiles,ldap,smtp]
 # On production the VISUAL check connects to VISUAL production, read-only, as visualkit_ro.
 set -euo pipefail
 PS_RUN="${PS_RUN:-/usr/local/sbin/ps-run}"
 exec "$PS_RUN" doctor "$@"
@@ -0,0 +1,73 @@
 #!/usr/bin/env bash
 # Packet Station launcher on a VM. The native counterpart of docker/entrypoint.sh.
 # The systemd units and ps-run start every packetstation command through it.
 #
 # Usage: ps-launch.sh app | worker | migrate | doctor [...] | job run <name> | version | healthcheck app|worker | ...
 #
 # It runs from a release folder, /opt/packetstation/<release>/deploy/vm/ps-launch.sh, and uses that
 # release's venv and built frontend. systemd has already loaded /etc/packetstation/packetstation.env.
 #
 # Secrets: each unit hands its own secrets to the service as systemd credentials (LoadCredential=),
 # so the app never sees the VISUAL or M-Files password. When $CREDENTIALS_DIRECTORY holds a known
 # secret, the matching *_FILE setting is pointed at it. MFilesKit reads its password from the
 # environment only, so MFILES_PASSWORD is exported from MFILES_PASSWORD_FILE, as the entrypoint does.
 # No value is ever printed.
 #
 # Set here, over the env file: STATIC_DIR and PS_BUILD (from the release), WORKER_BEAT_FILE, TZ, LANG.
 set -euo pipefail
 self="$(readlink -f -- "${BASH_SOURCE[0]}")"
 release="$(cd -- "$(dirname -- "$self")/../.." && pwd -P)"
 venv="${PS_VENV:-$release/venv}"
 # secret name (the credential id and the file name in /etc/packetstation/secrets) and its setting
 cred_map='postgres_app_password PG_APP_PASSWORD_FILE
 postgres_migrate_password PG_MIGRATE_PASSWORD_FILE
 visualkit_password VISUAL_PASSWORD_FILE
 mfileskit_password MFILES_PASSWORD_FILE
 ldap_bind_password LDAP_PASSWORD_FILE
 session_key SESSION_KEY_FILE
 seal_key SEAL_KEY_FILE'
 if [[ -n "${CREDENTIALS_DIRECTORY:-}" ]]; then
  while read -r name var; do
    if [[ -r "$CREDENTIALS_DIRECTORY/$name" ]]; then
      export "$var=$CREDENTIALS_DIRECTORY/$name"
    fi
  done <<< "$cred_map"
 fi
 unset cred_map name var
 mfiles_secret="${MFILES_PASSWORD_FILE:-}"
 if [[ -n "$mfiles_secret" && -r "$mfiles_secret" ]]; then
  MFILES_PASSWORD="$(<"$mfiles_secret")"
  # Drop a trailing carriage return left by an editor on Windows.
  MFILES_PASSWORD="${MFILES_PASSWORD%$'\r'}"
  export MFILES_PASSWORD
 fi
 unset mfiles_secret
 build="dev"
 if [[ -r "$release/BUILD" ]]; then
  build="$(head -n 1 -- "$release/BUILD")"
  build="${build%$'\r'}"
 fi
 export PATH="$venv/bin:/usr/local/bin:/usr/bin:/bin"
 export STATIC_DIR="$release/static"
 export PS_BUILD="$build"
 # The worker unit makes /run/packetstation (RuntimeDirectory=). The heartbeat lives there so
 # `ps-run healthcheck worker` reads the same file the worker writes.
 export WORKER_BEAT_FILE=/run/packetstation/worker.beat
 # The same as the containers: UTC process clock, UTF-8, unbuffered logs to the journal.
 export TZ=UTC
 export LANG=C.UTF-8
 export PYTHONUNBUFFERED=1
 export PYTHONDONTWRITEBYTECODE=1
 unset build self
 if [[ $# -eq 0 ]]; then
  set -- app
 fi
 exec "$venv/bin/packetstation" "$@"
@@ -0,0 +1,66 @@
 #!/usr/bin/env bash
 # Run one packetstation command on this VM the way the services run it: as the packetstation user,
 # with the env file, the secrets as systemd credentials and the same sandbox, through systemd-run.
 # Output comes back to this terminal and the exit code is the command's.
 #
 # Installed as /usr/local/sbin/ps-run by deploy/vm/install.sh. Run it as root (sudo).
 #
 # Usage:
 #   ps-run doctor [--skip visual,mfiles,ldap,smtp]
 #   ps-run job run visual-feed
 #   ps-run ad-groups --list
 #   ps-run healthcheck worker
 #   ps-run version
 #
 # Every secret file present in /etc/packetstation/secrets is handed over as a credential; the
 # folder itself stays out of reach of the command. No value is read or printed here.
 set -euo pipefail
 ENV_FILE="${PS_ENV_FILE_PATH:-/etc/packetstation/packetstation.env}"
 SECRETS_DIR="${PS_SECRETS_DIR:-/etc/packetstation/secrets}"
 CURRENT="${PS_CURRENT:-/opt/packetstation/current}"
 SYSTEMD_RUN="${SYSTEMD_RUN:-systemd-run}"
 SERVICE_USER="${PS_USER:-packetstation}"
 ASSETS_DIR_RW="${PS_ASSETS_DIR:-/var/lib/packetstation/assets}"
 ARCHIVE_DIR_RW="${PS_ARCHIVE_MOUNT:-/mnt/packetstation-archive}"
 die() { printf 'ps-run: %s\n' "$*" >&2; exit 2; }
 if [[ $# -eq 0 || "$1" == "-h" || "$1" == "--help" ]]; then
  sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' >&2
  exit 2
 fi
 if [[ "${PS_RUN_ALLOW_NONROOT:-}" != "1" && "$(id -u)" != "0" ]]; then
  die "run as root (sudo ps-run ...)"
 fi
 [[ -r "$ENV_FILE" ]] || die "env file $ENV_FILE is missing"
 [[ -x "$CURRENT/deploy/vm/ps-launch.sh" ]] || die "no release at $CURRENT (run ps-upgrade first)"
 props=(
  -p "User=$SERVICE_USER"
  -p "Group=$SERVICE_USER"
  -p "EnvironmentFile=$ENV_FILE"
  -p UMask=0027
  -p NoNewPrivileges=yes
  -p ProtectSystem=strict
  -p ProtectHome=yes
  -p PrivateTmp=yes
  -p PrivateDevices=yes
  -p ProtectKernelTunables=yes
  -p ProtectKernelModules=yes
  -p ProtectControlGroups=yes
  -p RestrictSUIDSGID=yes
  -p "ReadWritePaths=$ASSETS_DIR_RW"
  -p "ReadWritePaths=-$ARCHIVE_DIR_RW"
  -p "InaccessiblePaths=-$SECRETS_DIR"
 )
 for name in postgres_app_password postgres_migrate_password visualkit_password mfileskit_password \
            ldap_bind_password session_key seal_key; do
  if [[ -f "$SECRETS_DIR/$name" ]]; then
    props+=(-p "LoadCredential=$name:$SECRETS_DIR/$name")
  fi
 done
 exec "$SYSTEMD_RUN" --wait --pipe --collect --quiet --service-type=exec \
  --description="Packet Station: packetstation $1" \
  "${props[@]}" -- "$CURRENT/deploy/vm/ps-launch.sh" "$@"
@@ -0,0 +1,70 @@
 # Packet Station: the web app (uvicorn on port 8000, the API and the built frontend).
 # Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
 # NPM on docker-1 is the only client allowed on port 8000 (packetstation-firewall.service).
 [Unit]
 Description=Packet Station web app
 Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
 Requires=packetstation-migrate.service
 # Wants, not Requires: a restart of nftables.service must not restart the app. ExecStartPre checks the table.
 Wants=packetstation-firewall.service
 After=packetstation-migrate.service packetstation-firewall.service [email protected] network-online.target
 Wants=[email protected] network-online.target
 StartLimitIntervalSec=300
 StartLimitBurst=5
 [Service]
 Type=exec
 User=packetstation
 Group=packetstation
 EnvironmentFile=/etc/packetstation/packetstation.env
 # The app's secrets only. It never gets the VISUAL or M-Files password.
 LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
 LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
 LoadCredential=session_key:/etc/packetstation/secrets/session_key
 LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
 # The app never listens without the firewall table in place ("+": this check runs as root).
 ExecStartPre=+/usr/sbin/nft list table inet packetstation
 ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh app
 SyslogIdentifier=packetstation-app
 Restart=on-failure
 RestartSec=5s
 TimeoutStopSec=30s
 WorkingDirectory=/
 UMask=0027
 MemoryMax=1G
 LimitNOFILE=16384
 # Sandbox. Read-only file system except the assets and the archive mount.
 # The leading "-" lets the app start while the archive is not mounted (ARCHIVE_ROOT unset).
 ReadWritePaths=/var/lib/packetstation/assets
 ReadWritePaths=-/mnt/packetstation-archive
 InaccessiblePaths=/etc/packetstation/secrets
 NoNewPrivileges=yes
 ProtectSystem=strict
 ProtectHome=yes
 PrivateTmp=yes
 PrivateDevices=yes
 ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
 ProtectClock=yes
 ProtectHostname=yes
 ProtectProc=invisible
 # AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
 RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
 RestrictNamespaces=yes
 RestrictRealtime=yes
 RestrictSUIDSGID=yes
 LockPersonality=yes
 RemoveIPC=yes
 CapabilityBoundingSet=
 AmbientCapabilities=
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
 SystemCallErrorNumber=EPERM
 # Not set: MemoryDenyWriteExecute. pypdfium2 goes through ctypes, whose callbacks may need
 # writable and executable memory. Turn it on only after a filing and a print pass on the test VM.
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,21 @@
 # Packet Station: only NPM on docker-1 (TRUSTED_PROXY_HOST) and the VM itself may reach port 8000.
 # Loads /etc/packetstation/packetstation.nft, which deploy/vm/install.sh writes from the env file.
 # The table is Packet Station's own (inet packetstation); other rules on the VM are left alone.
 [Unit]
 Description=Packet Station firewall for the app port
 Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
 Before=network-pre.target packetstation-app.service
 Wants=network-pre.target
 # A restart of nftables.service flushes the whole ruleset. Load the table again after it.
 After=nftables.service
 PartOf=nftables.service
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
 ExecReload=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
 ExecStop=-/usr/sbin/nft delete table inet packetstation
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,52 @@
 # Packet Station: SQL migrations, once per boot and once per upgrade, before the app and the worker.
 # Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
 # ps-upgrade restarts this unit after switching /opt/packetstation/current to a new release.
 [Unit]
 Description=Packet Station database migrations
 Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
 Wants=[email protected] network-online.target
 After=[email protected] network-online.target
 Before=packetstation-app.service packetstation-worker.service
 ConditionPathExists=/opt/packetstation/current/deploy/vm/ps-launch.sh
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 User=packetstation
 Group=packetstation
 EnvironmentFile=/etc/packetstation/packetstation.env
 # migrate needs both passwords: it logs in as ps_migrate and grants to ps_app.
 LoadCredential=postgres_migrate_password:/etc/packetstation/secrets/postgres_migrate_password
 LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
 ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh migrate
 SyslogIdentifier=packetstation-migrate
 TimeoutStartSec=15min
 WorkingDirectory=/
 UMask=0027
 # Sandbox. The whole file system is read-only; migrate writes nothing outside PostgreSQL.
 NoNewPrivileges=yes
 ProtectSystem=strict
 ProtectHome=yes
 PrivateTmp=yes
 PrivateDevices=yes
 InaccessiblePaths=/etc/packetstation/secrets
 ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
 ProtectClock=yes
 ProtectHostname=yes
 ProtectProc=invisible
 RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
 RestrictNamespaces=yes
 RestrictRealtime=yes
 RestrictSUIDSGID=yes
 LockPersonality=yes
 RemoveIPC=yes
 CapabilityBoundingSet=
 AmbientCapabilities=
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
 SystemCallErrorNumber=EPERM
 # No [Install]: the app and the worker pull this unit in (Requires=).
@@ -0,0 +1,67 @@
 # Packet Station: the worker (VISUAL feed, drawings, AD reconcile, integrity, backup check,
 # notifications, archive copies). Installed by deploy/vm/install.sh into /etc/systemd/system.
 # On production its first start is Spencer's approval of the scheduled VISUAL polling.
 [Unit]
 Description=Packet Station worker
 Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
 Requires=packetstation-migrate.service
 After=packetstation-migrate.service [email protected] network-online.target
 Wants=[email protected] network-online.target
 StartLimitIntervalSec=300
 StartLimitBurst=5
 [Service]
 Type=exec
 User=packetstation
 Group=packetstation
 EnvironmentFile=/etc/packetstation/packetstation.env
 LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
 LoadCredential=visualkit_password:/etc/packetstation/secrets/visualkit_password
 LoadCredential=mfileskit_password:/etc/packetstation/secrets/mfileskit_password
 LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
 LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
 ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh worker
 SyslogIdentifier=packetstation-worker
 # /run/packetstation holds the heartbeat file (WORKER_BEAT_FILE, set by ps-launch.sh).
 RuntimeDirectory=packetstation
 RuntimeDirectoryMode=0755
 Restart=on-failure
 RestartSec=10s
 TimeoutStopSec=60s
 WorkingDirectory=/
 UMask=0027
 MemoryMax=1G
 # Sandbox. Read-only file system except the assets and the archive mount. The backup folder
 # (BACKUP_DIR) is only read, so it needs no entry here.
 ReadWritePaths=/var/lib/packetstation/assets
 ReadWritePaths=-/mnt/packetstation-archive
 InaccessiblePaths=/etc/packetstation/secrets
 NoNewPrivileges=yes
 ProtectSystem=strict
 ProtectHome=yes
 PrivateTmp=yes
 PrivateDevices=yes
 ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
 ProtectClock=yes
 ProtectHostname=yes
 ProtectProc=invisible
 # AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
 RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
 RestrictNamespaces=yes
 RestrictRealtime=yes
 RestrictSUIDSGID=yes
 LockPersonality=yes
 RemoveIPC=yes
 CapabilityBoundingSet=
 AmbientCapabilities=
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
 SystemCallErrorNumber=EPERM
 # Not set: MemoryDenyWriteExecute (see packetstation-app.service).
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,101 @@
 # Packet Station test settings on the test VM (native, no Docker).
 # deploy/vm/install.sh --stack test copies this to /etc/packetstation/packetstation.env
 # (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
 # It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
 # The test VM never points at VISUAL production. It reads VISUAL DEV.
 #
 # systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
 # Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
 # Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
 # ---- Stack and site ----
 # prod, test, ci or dev. Guards against crossing stacks.
 STACK=test
 # The VISUAL site in every call. The docker-1 test stack runs with PESCO.
 SITE_ID=PESCO
 # Site time zone for schedules and display. The processes run in UTC.
 SITE_TZ=America/Denver
 # Public URL, used in links and mail.
 PUBLIC_URL=https://packetstation-test.pescoinc.biz
 # NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
 # opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
 TRUSTED_PROXY_HOST=192.168.101.12
 # DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
 LOG_LEVEL=INFO
 # ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
 # A folder means the Unix socket. PostgreSQL listens on no TCP port.
 PG_HOST=/var/run/postgresql
 PG_PORT=5432
 PG_DATABASE=packetstation
 PG_APP_USER=ps_app
 PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
 PG_MIGRATE_USER=ps_migrate
 PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
 # ---- Keys and folders ----
 # The services get the secrets as systemd credentials and ps-launch.sh points these at them.
 # The paths here are the files those credentials are loaded from.
 SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
 SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
 # Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
 ASSETS_DIR=/var/lib/packetstation/assets
 # Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
 BACKUP_DIR=/var/backups/packetstation
 # The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
 # `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
 # Admin on the Admin page, File locations.
 #ARCHIVE_ROOT=/mnt/packetstation-archive
 # ---- Uploads and storage ----
 UPLOAD_MAX_BYTES=26214400
 UPLOAD_MAX_BYTES_BOM=5242880
 ASSETS_QUOTA_WARN_PCT=85
 # 0 measures the file system under ASSETS_DIR.
 ASSETS_QUOTA_BYTES=0
 # ---- ClamAV (optional, off: Gitea issue 19) ----
 # Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
 #CLAMAV_HOST=127.0.0.1
 CLAMAV_PORT=3310
 CLAMAV_TIMEOUT_SECONDS=30
 # ---- LDAP (PESCO Active Directory) ----
 LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
 LDAP_BASE_DN=DC=PESCO,DC=com
 LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
 LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
 LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
 LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
 # The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
 LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
 LDAP_STARTTLS=false
 LDAP_ALLOW_INSECURE=false
 # ---- Mail (relay app-03, no auth) ----
 SMTP_HOST=app-03.pesco.com
 SMTP_PORT=25
 SMTP_FROM=[email protected]
 SMTP_STARTTLS=true
 SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
 # The test VM never mails people about test data. The Admin "Send a test" still sends.
 NOTIFY_DRY_RUN=true
 # ---- VisualKit (VISUAL ERP, read only) ----
 # VISUAL DEV. The test VM never points at production. Single quotes keep the backslash.
 VISUAL_SERVER='vsql.pesco.com\DEV'
 VISUAL_DATABASE=PESCOINCDEV
 VISUAL_USER=visualkit_ro
 VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
 # no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
 # store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
 VISUAL_ENCRYPT=no
 VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
 # ---- MFilesKit (M-Files, read only) ----
 MFILES_URL=https://mfiles.pescoinc.biz/REST
 MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
 MFILES_USERNAME=mfileskitRO
 # ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
 MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
 # Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.
@@ -0,0 +1,280 @@
 #!/usr/bin/env bash
 # Install a Packet Station release on this VM and switch to it. Run by Spencer as root.
 # Installed as /usr/local/sbin/ps-upgrade by deploy/vm/install.sh.
 #
 # Usage:
 #   ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]
 #   ps-upgrade --rollback
 #
 # RELEASE.tar.gz comes from deploy/vm/make-release.sh. When RELEASE.tar.gz.sha256 lies next to it,
 # the archive is checked against it first.
 #
 # Steps:
 #   1. Unpack into /opt/packetstation/<build> and build its venv there: public pins from PyPI
 #      (server/requirements.txt), visualkit and mfileskit from the Gitea index with --index-url
 #      (server/requirements-pesco.txt), then the package itself with --no-deps; pip check.
 #      A release folder that is already complete is reused.
 #   2. A backup (ps-backup) when the database exists, unless --no-backup.
 #   3. Stop the app and the worker, point /opt/packetstation/current at the new release
 #      (previous keeps the old one), install its units and commands (install.sh --files-only).
 #   4. Run the migrations (restart packetstation-migrate.service). On failure: current goes back,
 #      the services stay stopped, and the restore command is printed.
 #   5. Start again whichever of the app and the worker ran before. A first install starts nothing
 #      unless --start: on production the first start is Spencer's approval of the VISUAL polling.
 #   6. Wait for /healthz, run ps-doctor (unless --no-doctor) and keep the newest 3 releases.
 #
 # --rollback points current back at previous and restarts what ran. Migrations are not undone:
 # roll back only to a release whose code works with the schema now in the database.
 #
 # Python: PYTHON (default python3.12) must be 3.12 or newer.
 set -euo pipefail
 umask 022
 OPT="${PS_OPT:-/opt/packetstation}"
 PYTHON="${PYTHON:-python3.12}"
 PESCO_INDEX_URL="${PESCO_INDEX_URL:-https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/}"
 PYPI_INDEX_URL="${PYPI_INDEX_URL:-https://pypi.org/simple/}"
 KEEP_RELEASES="${KEEP_RELEASES:-3}"
 HEALTH_URL="http://127.0.0.1:8000/healthz"
 SERVICES=(packetstation-app.service packetstation-worker.service)
 log() { printf 'ps-upgrade: %s\n' "$*" >&2; }
 die() { printf 'ps-upgrade: ERROR: %s\n' "$*" >&2; exit 1; }
 usage() { sed -n '2,31p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
 tarball=""
 start=0
 backup=1
 doctor=1
 rollback=0
 while [[ $# -gt 0 ]]; do
  case "$1" in
    --start) start=1; shift ;;
    --no-backup) backup=0; shift ;;
    --no-doctor) doctor=0; shift ;;
    --rollback) rollback=1; shift ;;
    -h|--help) usage ;;
    -*) log "unknown option $1"; usage ;;
    *) [[ -z "$tarball" ]] || usage; tarball="$1"; shift ;;
  esac
 done
 [[ "$(id -u)" == "0" ]] || die "run as root"
 [[ $rollback -eq 1 || -n "$tarball" ]] || usage
 [[ "$KEEP_RELEASES" =~ ^[1-9][0-9]*$ ]] || die "KEEP_RELEASES must be a positive whole number"
 exec 9>/run/lock/packetstation-upgrade.lock
 flock -n 9 || die "another ps-upgrade is running"
 # ------------------------------------------------------------------ helpers
 was_active=()
 remember_active() {
  local s
  was_active=()
  for s in "${SERVICES[@]}"; do
    if systemctl is-active --quiet "$s"; then was_active+=("$s"); fi
  done
 }
 switch_to() {
  # Atomic: a new link next to current, then rename over it.
  local target="$1"
  ln -sfn -- "$target" "$OPT/.current.new"
  mv -T -- "$OPT/.current.new" "$OPT/current"
 }
 wait_healthy() {
  local i
  for (( i = 0; i < 30; i++ )); do
    if curl -fsS --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
      log "the app answers $HEALTH_URL"
      return 0
    fi
    sleep 2
  done
  log "WARNING: the app did not answer $HEALTH_URL within 60 s: journalctl -u packetstation-app -n 50"
  return 1
 }
 start_again() {
  local -a units=("$@")
  if [[ ${#units[@]} -eq 0 ]]; then
    log "the app and the worker were not running and stay stopped."
    log "start them with: systemctl enable --now packetstation-app packetstation-worker"
    return 0
  fi
  log "starting ${units[*]}"
  systemctl start "${units[@]}"
  local s
  for s in "${units[@]}"; do
    if [[ "$s" == packetstation-app.service ]]; then wait_healthy || true; fi
  done
 }
 prune() {
  local cur prev d
  cur="$(readlink -f -- "$OPT/current" 2>/dev/null || true)"
  prev="$(readlink -f -- "$OPT/previous" 2>/dev/null || true)"
  local -a dirs=()
  while IFS= read -r d; do dirs+=("$d"); done < <(
    find "$OPT" -mindepth 1 -maxdepth 1 -type d -name '[A-Za-z0-9]*' -printf '%T@ %p\n' | sort -rn | cut -d' ' -f2-
  )
  # current and previous always stay; they count toward KEEP_RELEASES.
  local allowed=$(( KEEP_RELEASES - 2 )) kept=0
  for d in "${dirs[@]}"; do
    if [[ "$d" == "$cur" || "$d" == "$prev" ]]; then continue; fi
    kept=$(( kept + 1 ))
    if [[ $kept -gt $allowed ]]; then
      log "removing old release $(basename -- "$d")"
      rm -rf -- "$d"
    fi
  done
 }
 # ------------------------------------------------------------------ rollback
 if [[ $rollback -eq 1 ]]; then
  [[ -L "$OPT/previous" ]] || die "no previous release to go back to"
  prev="$(readlink -f -- "$OPT/previous")"
  cur="$(readlink -f -- "$OPT/current")"
  [[ -f "$prev/.ready" ]] || die "previous release $prev is not complete"
  remember_active
  log "rolling back from $(basename -- "$cur") to $(basename -- "$prev")"
  if [[ ${#was_active[@]} -gt 0 ]]; then systemctl stop "${was_active[@]}"; fi
  switch_to "$prev"
  ln -sfn -- "$cur" "$OPT/previous"
  bash "$prev/deploy/vm/install.sh" --files-only
  start_again "${was_active[@]}"
  log "done. Migrations were not undone."
  exit 0
 fi
 # ------------------------------------------------------------------ 1. unpack and build
 [[ -r "$tarball" ]] || die "cannot read $tarball"
 if [[ -r "$tarball.sha256" ]]; then
  ( cd -- "$(dirname -- "$tarball")" && sha256sum --quiet -c -- "$(basename -- "$tarball").sha256" ) \
    || die "$tarball does not match $tarball.sha256"
  log "checksum ok"
 else
  log "no $tarball.sha256 next to the archive; checksum not checked"
 fi
 # Every member must sit under one top folder packetstation-<build>/, with no absolute path and no "..".
 # The listing is read whole first: an early-exiting reader (head, grep -q) would make tar fail
 # with SIGPIPE under pipefail.
 members="$(tar -tzf "$tarball")" || die "cannot list $tarball"
 top="${members%%$'\n'*}"
 top="${top%%/*}"
 [[ "$top" =~ ^packetstation-([A-Za-z0-9][A-Za-z0-9._-]{0,63})$ ]] || die "unexpected top folder '$top' in $tarball"
 build="${BASH_REMATCH[1]}"
 outside="$(grep -Evc "^$top(/|$)" <<< "$members" || true)"
 [[ "$outside" == "0" ]] || die "$tarball has files outside $top/"
 dotdot="$(grep -Ec '(^|/)\.\.(/|$)' <<< "$members" || true)"
 [[ "$dotdot" == "0" ]] || die "$tarball has a '..' path"
 unset members
 dest="$OPT/$build"
 if [[ -f "$dest/.ready" ]]; then
  log "release $build is already built in $dest"
 else
  "$PYTHON" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)' \
    || die "$PYTHON is not Python 3.12 or newer (set PYTHON=/path/to/python3.12)"
  rm -rf -- "$dest"
  install -d -o root -g root -m 0755 "$dest"
  log "unpacking $build into $dest"
  tar -xzf "$tarball" -C "$dest" --strip-components=1 --no-same-owner --no-same-permissions
  [[ "$(head -n 1 "$dest/BUILD")" == "$build" ]] || die "BUILD in the archive does not say $build"
  [[ -f "$dest/static/index.html" ]] || die "the archive has no built frontend (static/index.html)"
  log "building the venv"
  "$PYTHON" -m venv "$dest/venv"
  pip=("$dest/venv/bin/python" -m pip --disable-pip-version-check --no-input)
  # No pip.conf or PIP_* setting from the host may add an index.
  export PIP_CONFIG_FILE=/dev/null
  unset PIP_INDEX_URL PIP_EXTRA_INDEX_URL PIP_FIND_LINKS
  "${pip[@]}" install --index-url "$PYPI_INDEX_URL" -r "$dest/server/requirements.txt"
  # PESCO libraries from the Gitea index only. --index-url, never --extra-index-url.
  "${pip[@]}" install --index-url "$PESCO_INDEX_URL" -r "$dest/server/requirements-pesco.txt"
  # The package itself. The build backend (hatchling) comes from PyPI.
  "${pip[@]}" install --index-url "$PYPI_INDEX_URL" --no-deps "$dest"
  "${pip[@]}" check
  # The release is read-only to the services. Owned by root, nothing writable by others.
  chown -R root:root "$dest"
  chmod -R u+rwX,go+rX,go-w "$dest"
  # The units and ps-run start these directly; an archive made on Windows may have lost the x bit.
  chmod 0755 "$dest"/deploy/vm/*.sh "$dest"/scripts/backup/*.sh
  printf '%s\n' "$build" > "$dest/.ready"
  log "release $build built"
 fi
 # ------------------------------------------------------------------ 2. backup
 first_install=1
 [[ -L "$OPT/current" ]] && first_install=0
 db_exists=0
 found="$(setpriv --reuid=postgres --regid=postgres --init-groups -- \
  psql -X -At -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'packetstation'" 2>/dev/null || true)"
 if [[ "$found" == "1" ]]; then
  db_exists=1
 fi
 if [[ $backup -eq 1 && $db_exists -eq 1 && $first_install -eq 0 ]]; then
  log "backup before the upgrade"
  /usr/local/sbin/ps-backup || die "the backup failed; nothing was changed. Fix it, or use --no-backup."
 fi
 # ------------------------------------------------------------------ 3. switch
 remember_active
 if [[ $start -eq 1 ]]; then was_active=("${SERVICES[@]}"); fi
 old=""
 if [[ $first_install -eq 0 ]]; then
  old="$(readlink -f -- "$OPT/current")"
 fi
 if [[ "$old" == "$dest" ]]; then
  log "release $build is already current"
 fi
 active_now=()
 for s in "${SERVICES[@]}"; do
  if systemctl is-active --quiet "$s"; then active_now+=("$s"); fi
 done
 if [[ ${#active_now[@]} -gt 0 ]]; then
  log "stopping ${active_now[*]}"
  systemctl stop "${active_now[@]}"
 fi
 switch_to "$dest"
 if [[ -n "$old" && "$old" != "$dest" ]]; then
  ln -sfn -- "$old" "$OPT/previous"
 fi
 log "current is $build"
 bash "$dest/deploy/vm/install.sh" --files-only
 # ------------------------------------------------------------------ 4. migrate
 log "running the migrations"
 if ! systemctl restart packetstation-migrate.service; then
  journalctl -u packetstation-migrate.service -n 30 --no-pager >&2 || true
  if [[ -n "$old" && "$old" != "$dest" ]]; then
    switch_to "$old"
    bash "$old/deploy/vm/install.sh" --files-only || true
    log "current is back on $(basename -- "$old")"
  fi
  last="$(find /var/backups/packetstation -maxdepth 1 -name 'ps-*.dump' -printf '%f\n' 2>/dev/null | sort | tail -n 1)"
  die "the migrations failed. The app and the worker are stopped. Read the log above. If the database
 must go back to the backup taken before this upgrade, restore it (docs/DEPLOY.md, \"Restore on a VM\"):
  ps-restore --dump /var/backups/packetstation/${last:-<dump>} --db ps_restored"
 fi
 # ------------------------------------------------------------------ 5. and 6. start, check, prune
 start_again "${was_active[@]}"
 if [[ $doctor -eq 1 ]]; then
  log "doctor"
  /usr/local/sbin/ps-doctor || log "WARNING: the doctor reported a failure (see above)"
 fi
 prune
 log "done: $build"
@@ -1,6 +1,6 @@
 # Deploying Packet Station
-How Packet Station runs on `docker-1`. Sections not built yet are marked "planned, phase N". The design is in [PLAN.md](PLAN.md) under "Deployment and CI".
+How Packet Station runs. Production and test move to their own native Linux VMs in phase 10 ("Native VM install"). Until the test VM is ready the test stack runs in Docker on `docker-1`, which keeps CI. Sections not built yet are marked "planned, phase N". The design is in [PLAN.md](PLAN.md) under "Deployment and CI".
 ## Host facts
@@ -59,7 +59,7 @@ The PESCO CA chain is mounted into the containers once and referenced by `LDAP_C
 ## Env files
-One folder per stack. The test stack: `/home/spencerm/packetstation/test/`, like the other compose stacks on docker-1 (decided 2026-10-18, Gitea issue 38). It holds `stack.env`, `secrets/`, `ca/pesco-ca.pem` and `backups/`, and its `stack.env` sets `ENV_FILE`, `SECRETS_DIR`, `CA_BUNDLE_FILE` and `BACKUP_DIR` to those paths. The compose files run from the checkout in `/home/spencerm/packetstation/repo/deploy`. On 2026-10-27 Spencer decided that production and test each run natively on their own Linux VM, without Docker, and that docker-1 keeps CI only (Gitea issue 37). The test stack stays here until the test VM is ready. The production lines in this file describe the old compose plan and are rewritten for the VMs in phase 10. Each folder holds `stack.env` (settings only, no secrets) and `secrets/`. The env file sets `PROJECT_NAME`, `SECRETS_DIR` and `APP_ALIAS`, which the compose files require. `.env.example` in the repo lists every setting.
+One folder per stack. The test stack: `/home/spencerm/packetstation/test/`, like the other compose stacks on docker-1 (decided 2026-10-18, Gitea issue 38). It holds `stack.env`, `secrets/`, `ca/pesco-ca.pem` and `backups/`, and its `stack.env` sets `ENV_FILE`, `SECRETS_DIR`, `CA_BUNDLE_FILE` and `BACKUP_DIR` to those paths. The compose files run from the checkout in `/home/spencerm/packetstation/repo/deploy`. On 2026-10-27 Spencer decided that production and test each run natively on their own Linux VM, without Docker, and that docker-1 keeps CI only (Gitea issue 37). The test stack stays here until the test VM is ready. The VMs keep their settings in `/etc/packetstation/packetstation.env` and their secrets in `/etc/packetstation/secrets/`: see "Native VM install". The production compose lines in this file are the old plan, kept for reference. Each folder holds `stack.env` (settings only, no secrets) and `secrets/`. The env file sets `PROJECT_NAME`, `SECRETS_DIR` and `APP_ALIAS`, which the compose files require. `.env.example` in the repo lists every setting.
 Key settings:
@@ -132,7 +132,7 @@ The Gitea PyPI index `https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/
 ## NPM proxy host
-One proxy host per stack in Nginx Proxy Manager. To be recorded under `deploy/` as well.
+One proxy host per stack in Nginx Proxy Manager, recorded in `deploy/npm-proxy-host.md`. For the VMs see "NPM proxy host for the VMs": only the forward target and `TRUSTED_PROXY_HOST` change.
 - Domain: `packetstation-test.pescoinc.biz` (test) or `packetstation.pescoinc.biz` (production).
 - Scheme `http`, forward host the stack's alias (`packetstation-test-app` or `packetstation-app`), port `8000`.
@@ -241,7 +241,7 @@ Docker's default address pool on docker-1 holds fifteen /16 bridge networks (172
 ## Doctor
-`packetstation doctor` proves a host and a stack are ready. Run it with `compose run`, not `compose exec`: the entrypoint exports MFILES_PASSWORD from its secret file, and `exec` skips the entrypoint, so the M-Files password check fails under `exec` even though the running worker has it.
+`packetstation doctor` proves a host and a stack are ready. On a VM run `sudo ps-doctor` ("Commands on a VM"). On docker-1 run it with `compose run`, not `compose exec`: the entrypoint exports MFILES_PASSWORD from its secret file, and `exec` skips the entrypoint, so the M-Files password check fails under `exec` even though the running worker has it.
 ```bash
 docker compose -p packetstation-test --env-file ~/packetstation/test/stack.env \
@@ -340,113 +340,289 @@ The app sends test messages and the worker sends everything else, through PESCO'
 The doctor's SMTP line checks STARTTLS with certificate verification, without sending.
-## Backup (phase 9)
+## Native VM install (phase 10)
-Host scripts in `scripts/backup/`, run by root from a systemd timer. They run outside the app, so a broken app still backs up. [scripts/backup/README.md](../scripts/backup/README.md) has the same steps with more detail.
+Production and test each run natively on their own Linux VM, without Docker (Spencer, 2026-10-27, Gitea issue 37). docker-1 keeps CI only. The test VM is built first from the same steps, so the install, the backups, the restore and the doctor are rehearsed before production. The docker-1 test stack keeps running until the test VM is ready (see "Retiring the docker-1 test stack").
-| File | Installed as |
+The kit is in `deploy/vm/`. Spencer runs everything that needs root. Nothing in it starts the worker by itself: on production, Spencer's first start of the worker is the approval of the scheduled VISUAL polling.
 ### What IT provides (open on issue 37)
 - One VM per stack. Ubuntu 24.04 LTS is recommended: it ships Python 3.12, which the server needs (`requires-python >= 3.12`, pins resolved for 3.12). Debian 12 ships Python 3.11, so on Debian 12 a Python 3.12 has to be provided separately and passed to `ps-upgrade` as `PYTHON=/path/to/python3.12`. `install.sh` supports both and refuses anything else. Size: 2 vCPU, 4 GB RAM and 40 GB disk is a starting point (the app and the worker are capped at 1 GB each; the test stack's assets were 125 MB after a month of testing).
 - A fixed LAN address per VM. NPM forwards to it.
 - Outbound from the VM: the OS mirrors, `apt.postgresql.org`, `packages.microsoft.com`, `www.postgresql.org` (the PGDG key), `pypi.org` and `files.pythonhosted.org`, `gt.pescoinc.biz` (the Python index), and at run time vsql (1433, plus UDP 1434 for the named DEV instance), `mfiles.pescoinc.biz` 443, DC-02 and PDC-1 636, `app-03.pesco.com` 25, the QNAP (NFS 2049).
 - Inbound: SSH for Spencer, and port 8000 from docker-1 (192.168.101.12) only. The VM enforces the second itself.
 - PBS backs the VM up like any other, at 03:00.
 ### Layout on a VM
 | Path | Owner, mode | What |
 |---|---|---|
 | `/opt/packetstation/<build>/` | root, 0755 | One release: the source, `BUILD`, `static/` (the built frontend) and `venv/`. Read-only to the services. |
 | `/opt/packetstation/current`, `previous` | root | Symlinks to the running and the last release. The units start `current/deploy/vm/ps-launch.sh`. |
 | `/etc/packetstation/packetstation.env` | root:packetstation 0640 | Settings, no secrets. From `deploy/vm/prod.env.example` or `test.env.example`. |
 | `/etc/packetstation/secrets/` | root:packetstation 0750, files 0440 | One file per secret (table below). The services get them as systemd credentials and cannot open the folder. |
 | `/etc/packetstation/ca/pesco-ca.pem` | root, 0644 | The PESCO CA chain for `LDAP_CA_BUNDLE` and `SMTP_CA_BUNDLE`. Also in the system trust store, which the ODBC driver uses. |
 | `/etc/packetstation/backup.conf`, `packetstation.nft` | root, 0644 | The backup stack line; the firewall table (written by `install.sh`). |
 | `/var/lib/packetstation/assets/` | packetstation, 0750 | `ASSETS_DIR`, the stored files. |
 | `/var/backups/packetstation/` | root:packetstation 0750 | `BACKUP_DIR`, the nightly dumps and markers. |
 | `/mnt/packetstation-archive/` | root, 0755 until mounted | `ARCHIVE_ROOT`, the QNAP NFS share. |
 | `/usr/local/sbin/ps-run`, `ps-doctor`, `ps-upgrade`, `ps-backup`, `ps-restore` | root, 0755 | The commands below. |
 The service user is `packetstation`, uid and gid 10001: the image's uid, so the NFS share's rights and the old volume's file owners carry over. PostgreSQL 16 from the PGDG repository listens on the local socket only (`listen_addresses = ''`, server time zone UTC, `conf.d/packetstation.conf`). `pg_hba.conf` gets one block, before Debian's peer line, that lets `ps_app` and `ps_migrate` log in to `packetstation` on the socket with their passwords (scram-sha-256). `install.sh` creates the two roles and the database as `deploy/postgres/initdb` does (owner `ps_migrate`, UTF8, locale `en_US.UTF-8`, `template0`, CONNECT and TEMPORARY for `ps_app`, schema `public` owned by `ps_migrate`). The env file sets `PG_HOST=/var/run/postgresql`.
 | Secret file | Made by | Given to |
 |---|---|---|
 | `postgres_migrate_password` | `install.sh --generate-secrets` | migrate |
 | `postgres_app_password` | `install.sh --generate-secrets` | migrate, app, worker |
 | `session_key` | `install.sh --generate-secrets` | app |
 | `seal_key` | `install.sh --generate-secrets` | app, worker |
 | `visualkit_password` | Spencer | worker |
 | `mfileskit_password` | Spencer | worker (`ps-launch.sh` exports `MFILES_PASSWORD`) |
 | `ldap_bind_password` | Spencer | app, worker |
 There is no PostgreSQL superuser password on a VM: root reaches the `postgres` OS user, which PostgreSQL trusts on the socket. `ps-run` hands every secret file present to the command it runs.
 ### The units
 | Unit | What |
 |---|---|
-| `ps-backup.sh` | `/usr/local/sbin/ps-backup.sh` (root, 0755) |
+| `packetstation-migrate.service` | Oneshot, `RemainAfterExit`: `packetstation migrate` as `ps_migrate`. Pulled in by the app and the worker (`Requires=`), so neither starts on a failed migration. `ps-upgrade` restarts it. |
-| `ps-restore.sh` | `/usr/local/sbin/ps-restore.sh` (root, 0755) |
+| `packetstation-app.service` | `packetstation app`, uvicorn on port 8000. Refuses to start without the firewall table. `Restart=on-failure`, `MemoryMax=1G`. |
-| `backup.conf.example` | `/etc/packetstation/backup.conf` (root, 0644) |
+| `packetstation-worker.service` | `packetstation worker`. Heartbeat in `/run/packetstation/worker.beat`. `Restart=on-failure`, `MemoryMax=1G`. |
-| `packetstation-backup.service` | `/etc/systemd/system/` (root, 0644) |
+| `packetstation-firewall.service` | Loads `/etc/packetstation/packetstation.nft`: its own table `inet packetstation`, which lets only `TRUSTED_PROXY_HOST` (NPM) and the VM itself reach port 8000. Nothing else on the VM is filtered by it. |
-| `packetstation-backup.timer` | `/etc/systemd/system/` (root, 0644) |
+| `packetstation-backup.service`, `.timer` | `ps-backup` at 02:00 America/Denver. |
-`backup.conf` lists the stacks, one per line: name, container (or `project:<compose project>`), database, folder.
+The app, the worker and migrate run as `packetstation` with `EnvironmentFile=/etc/packetstation/packetstation.env` and a sandbox: `ProtectSystem=strict` (writable only `/var/lib/packetstation/assets` and `/mnt/packetstation-archive`, the latter optional so the app starts without the share), `ProtectHome`, `PrivateTmp`, `PrivateDevices`, `NoNewPrivileges`, no capabilities, `InaccessiblePaths=/etc/packetstation/secrets`, `SystemCallFilter=@system-service`, the address families `AF_UNIX AF_INET AF_INET6 AF_NETLINK`, and the kernel protections. `systemd-analyze security` rates the app 1.5 ("OK"). `MemoryDenyWriteExecute` is left off: pypdfium2 uses ctypes, whose callbacks may need writable executable memory. Try it on the test VM after a filing and a print pass before turning it on.
-```
+Each unit loads only its own secrets with `LoadCredential=`, as compose did with secrets: the app never gets the VISUAL or M-Files password, and migrate gets only the two database passwords. `deploy/vm/ps-launch.sh` (the native counterpart of `docker/entrypoint.sh`) points each `*_FILE` setting at the credential, exports `MFILES_PASSWORD` from its file, and sets `STATIC_DIR` and `PS_BUILD` from the release, `WORKER_BEAT_FILE`, `TZ=UTC` and `LANG=C.UTF-8`. Those five are ignored in the env file.
 test     project:packetstation-test   packetstation  /home/spencerm/packetstation/test/backups
 #prod    project:packetstation        packetstation  /var/backups/packetstation/prod
 ```
-The production line stays commented out until the production stack runs (phase 10). Until then it would fail every night.
+Logs go to the journal: `journalctl -u packetstation-app -u packetstation-worker -f`.
-Where the dumps go:
+### Commands on a VM
- Test stack: `/home/spencerm/packetstation/test/backups`, owned by spencerm (the script gives new files the folder's owner and group). The test `stack.env` sets `BACKUP_DIR` to this path.
+| Command (as root) | What |
- Production: `/var/backups/packetstation/prod`, `root:10001` 0750. The production `stack.env` sets `BACKUP_DIR=/var/backups/packetstation/prod`.
+|---|---|
 | `ps-doctor [--skip visual,mfiles,ldap,smtp,archive]` | `packetstation doctor`, run the way the services run. |
 | `ps-run <packetstation command>` | Any `packetstation` command as the service user, with the env file, the secrets as credentials and the sandbox, through `systemd-run`. For example `ps-run job run visual-feed`, `ps-run ad-groups --list`, `ps-run healthcheck worker`, `ps-run version`. |
 | `ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]` | Install a release and switch to it (below). |
 | `ps-upgrade --rollback` | Back to the previous release. |
 | `ps-backup`, `ps-restore` | "Backups on a VM", "Restore on a VM". |
-The worker mounts the stack's folder read-only at `/backups`. Its `backup-check` job (07:15) only lists the folder, reads the dumps' sizes and times, and reads the markers.
+On production the doctor's VISUAL check connects to VISUAL production, read-only, as `visualkit_ro`. Use `--skip visual` before the go-live approval.
-What a run does, per stack:
+### Release archives
-1. Finds the stack's running postgres container (by compose labels for `project:`). None or two is a failure.
+A VM never builds the frontend and never runs Docker. `deploy/vm/make-release.sh` runs on a build host as a normal user, in a git checkout, and writes `packetstation-<build>.tar.gz` with a `.sha256`: the source of one commit (`pyproject.toml`, `README.md`, `server/`, `deploy/`, `scripts/`, `docs/`, with LF line endings), `BUILD`, and `static/`. Python packages are not in it.
 2. `docker exec <pg> pg_dump -U postgres -d packetstation -Fc` over the local socket, which the postgres image trusts. No password is read or passed. Written under a temporary name, `.ps-<stamp>.partial`.
 3. `pg_dumpall --globals-only --no-role-passwords`: the roles without their password hashes. The role passwords are in the stack secrets.
 4. `pg_restore --list` with the container's own pg_restore must list at least one entry. Then the files are renamed to `ps-<UTC yyyymmddThhmmssZ>.dump` and `.globals.sql`, mode 0600.
 5. Keeps the newest 7 (`KEEP=7` in the unit, `--keep N`), with their globals files.
 6. Writes `last-success` or `last-failure` (`<ISO UTC time> <dump name>`, plus the reason on failure), mode 0644. A failed run leaves no partial file.
-Each stack runs on its own, so one failing does not stop the next. Exit 1 if any stack failed, 2 for a usage or config error. `flock` keeps two runs from overlapping.
+- `--image gt.pescoinc.biz/spencerm/packetstation:sha-<sha>` (on docker-1): copies `/app/static` out of the CI-tested image with `docker create` and `docker cp` (nothing runs) and takes the commit from the image's build label. The release is then the tested commit with the tested frontend. This is the normal way.
 - `--npm [--ref REF]` (on a PC with Node): builds the frontend with `npm ci` and `npm run build` from the same commit.
-The unit is `Type=oneshot` as root, `Nice=10`, `IOSchedulingClass=idle`, `TimeoutStartSec=1h`, with `NoNewPrivileges`, `PrivateTmp` and `ProtectSystem=full` (no `ProtectHome`, so it can write the test folder under `/home`). The timer is `OnCalendar=*-*-* 02:00:00 America/Denver` with `Persistent=true`: 02:00 Denver time all year, an hour before the 03:00 PBS run, so PBS captures the fresh dumps. A missed run runs at the next boot.
+`ps-upgrade` builds the venv on the VM, as the image does: `server/requirements.txt` from PyPI with `--index-url https://pypi.org/simple/`, `server/requirements-pesco.txt` from the Gitea index with `--index-url` (never `--extra-index-url`), then the package with `--no-deps`, then `pip check`. Host pip settings are ignored (`PIP_CONFIG_FILE=/dev/null`).
-### Install (Spencer, with sudo, on docker-1)
+### First install (Spencer, on a new VM)
-After the phase 9 tree is synced to `~/packetstation/repo`:
+The test VM first. `<build>` is the release, for example `sha-<commit>`.
 1. On docker-1, in a git checkout of the repository that has the commit (for example a clone of `gt.pescoinc.biz/spencerm/PacketStation`):
   ```bash
   deploy/vm/make-release.sh --image gt.pescoinc.biz/spencerm/packetstation:sha-<sha> --out ~/releases
   ```
 2. Copy `~/releases/packetstation-<build>.tar.gz`, its `.sha256` and the PESCO CA bundle to the VM. The bundle comes from `scripts/Export-PescoCaBundle.ps1 -NoCopy` (or `-Remote` with the VM's path), or from `~/packetstation/test/ca/pesco-ca.pem` on docker-1.
 3. On the VM, unpack and run the install. It installs the packages, makes the user, folders, env file, CA, the four internal secrets, PostgreSQL's settings, roles and database, the units, the commands, the backup timer and the firewall:
   ```bash
   cd ~ && sha256sum -c packetstation-<build>.tar.gz.sha256
   tar -xzf packetstation-<build>.tar.gz
   sudo bash ./packetstation-<build>/deploy/vm/install.sh --stack test --ca-bundle ~/pesco-ca.pem --generate-secrets
   ```
   For production, `--stack prod`. If the VM is to carry on the docker-1 test stack's data, copy that stack's `seal_key` and `session_key` into `/etc/packetstation/secrets/` before this step instead of generating them: filed packets are sealed with the seal key.
 4. Put the three outside secrets in place, without echo, then run the install again so it checks them and sets their owner and mode:
   ```bash
   for s in visualkit_password mfileskit_password ldap_bind_password; do
     sudo bash -c 'umask 077; read -rsp "$1: " v; echo; printf %s "$v" > "/etc/packetstation/secrets/$1"' _ "$s"
   done
   sudo bash ./packetstation-<build>/deploy/vm/install.sh --skip-packages
   ```
 5. Read `/etc/packetstation/packetstation.env` through (`sudoedit`). Check `STACK`, `PUBLIC_URL`, `VISUAL_SERVER` and `VISUAL_DATABASE` (test: `'vsql.pesco.com\DEV'` and `PESCOINCDEV`; production: `vsql.pesco.com` and `PESCOINC`), `NOTIFY_DRY_RUN` (test `true`), and `TRUSTED_PROXY_HOST=192.168.101.12`. After changing `TRUSTED_PROXY_HOST`, run `install.sh --files-only` to rewrite the firewall table.
 6. Install the release. It builds the venv, runs the migrations and the doctor, and starts nothing on a first install:
   ```bash
   sudo ps-upgrade ~/packetstation-<build>.tar.gz
   ```
 7. Run the doctor until it is clean (on production with `--skip visual` until the approval):
   ```bash
   sudo ps-doctor
   ```
 8. The archive share, when the QNAP export exists ("Archive NFS mount").
 9. Start. On production this is the approval of the scheduled VISUAL polling:
   ```bash
   sudo systemctl enable --now packetstation-app packetstation-worker
   systemctl status packetstation-app packetstation-worker --no-pager
   curl -fsS http://127.0.0.1:8000/healthz
   ```
 10. Point the NPM proxy host at the VM ("NPM proxy host for the VMs") and run its checks.
 11. Backups: `sudo systemctl start packetstation-backup.service`, then `sudo ls -la /var/backups/packetstation`. Then rehearse a restore ("Restore on a VM").
 On Ubuntu with `ufw` turned on, ufw also has to let docker-1 in: `sudo ufw allow from 192.168.101.12 to any port 8000 proto tcp`. The Packet Station table only narrows port 8000 further; it never opens anything ufw blocks.
 ### Upgrades
 Build the archive for the new commit (step 1), copy it over (step 2), then:
 ```bash
 sudo ps-upgrade ~/packetstation-<build>.tar.gz
 ```
 In order: check the `.sha256` when it lies next to the archive, unpack into `/opt/packetstation/<build>` and build its venv (an already complete release folder is reused), take a backup with `ps-backup`, stop the app and the worker, point `current` at the new release (`previous` keeps the old), install its units and commands (`install.sh --files-only`), run the migrations, start again whichever of the two ran before, wait for `/healthz`, run the doctor, and keep the newest three releases. A short downtime window, as with a new image.
 If the migrations fail, `current` goes back to the old release, the app and the worker stay stopped, and the restore command for the backup taken a minute earlier is printed. Read `journalctl -u packetstation-migrate`.
 `sudo ps-upgrade --rollback` points `current` back at `previous` and restarts what ran. It does not undo migrations: roll back only to a release whose code works with the schema in the database, or restore the backup.
 Open screens see the build change and offer a reload (the version banner).
 ### NPM proxy host for the VMs
 NPM stays on docker-1 and keeps the certificates. The DNS names keep pointing at docker-1 (192.168.101.12). Only the forward target of each proxy host changes, to its VM over the LAN:
 | | Production | Test |
 |---|---|---|
 | Domain name | `packetstation.pescoinc.biz` | `packetstation-test.pescoinc.biz` |
 | Scheme | `http` | `http` |
 | Forward hostname / IP | the production VM's address | the test VM's address |
 | Forward port | `8000` | `8000` |
 | `TRUSTED_PROXY_HOST` on the VM | `192.168.101.12` | `192.168.101.12` |
 Everything else is as before (`deploy/npm-proxy-host.md`): Cache assets off, Block common exploits on, Websockets off, no access list; the PESCO certificate, Force SSL on, HTTP/2 on, NPM HSTS off (the app sends `Strict-Transport-Security` itself; two headers would conflict); and the same Advanced block, unchanged:
 ```nginx
 client_max_body_size 30m;
 location /api/events {
    proxy_pass $forward_scheme://$server:$port;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_set_header Host $host;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_buffering off;
    proxy_cache off;
    proxy_read_timeout 3600s;
    proxy_send_timeout 3600s;
 }
 ```
 With a fixed VM address the forward target can be the address itself; the "never an IP" rule was for container addresses, which change. `$server` keeps the events location on the same target as the rest of the host.
 Trust: NPM's container reaches the VM through docker-1's own address, so the VM sees every proxied request from 192.168.101.12. `TRUSTED_PROXY_HOST=192.168.101.12` makes the app trust `X-Forwarded-For` and `X-Forwarded-Proto` from that address only (`proxy.py` resolves an address literal to itself; no server change was needed), and the firewall table lets only that address reach port 8000. Other hosts on the LAN cannot reach the app directly, and so cannot fake a client address for the sign-in rate limit. Traffic from NPM to the VM is plain HTTP on the LAN; TLS to the VM is still open on issue 37.
 Check after switching: the four checks in `deploy/npm-proxy-host.md`, plus from another LAN PC `curl -m 5 http://<vm>:8000/healthz` must time out (the firewall), and the Admin page's access log shows the real client addresses, not 192.168.101.12.
 ### Archive NFS mount
 The archive share (Gitea issue 7) lives on the QNAP. Spencer creates the NFS export with read and write for the VMs' addresses and makes the share's folder owned by uid 10001 and gid 10001 (or maps the VM's uid 10001 to an owner with write rights). Root squash on is fine: only `packetstation` writes there.
 1. `nfs-common` is installed by `install.sh`. Add one line to `/etc/fstab` (the QNAP's name and export path from IT):
   ```
   <qnap>:/<export>  /mnt/packetstation-archive  nfs4  rw,soft,timeo=150,retrans=3,noatime,nosuid,nodev,noexec,_netdev,nofail  0  0
   ```
   `soft`: an unreachable QNAP makes a copy fail after about a minute instead of hanging the worker for good. The archive job writes each file to a temporary name, fsyncs and renames it, so a failed write shows up as a failed copy and is tried again with backoff. Filing never waits on it. `nofail` lets the VM boot without the QNAP.
 2. Mount it and make the archive folder as the service user:
   ```bash
   sudo mount /mnt/packetstation-archive
   sudo setpriv --reuid=packetstation --regid=packetstation --init-groups mkdir /mnt/packetstation-archive/packets
   ```
 3. Set `ARCHIVE_ROOT=/mnt/packetstation-archive` in the env file (uncomment the line) and restart: `sudo systemctl restart packetstation-app packetstation-worker`.
 4. On the Admin page, File locations, enter `packets`, press Check, then Save. `sudo ps-doctor` reports `archive` PASS from then on.
 While the share is not mounted, the mount point is root's and read-only to `packetstation`, and `packets` does not exist under it, so nothing is written to the VM's own disk and the check fails with a plain message. The units list the mount point as an optional writable path, so the app and the worker start without it.
 ### Retiring the docker-1 test stack
 The docker-1 test stack keeps running until the test VM passes its doctor, the NPM switch, a backup and a restore rehearsal. To carry its data over instead of starting empty: take a dump with the backup already installed on docker-1, tar the assets volume as in the 2026-10-21 rehearsal, copy both and the `seal_key` and `session_key` to the VM, and restore with `ps-restore` ("Restore on a VM"). Then on docker-1: stop the stack (`docker compose -p packetstation-test ... down`, keeping its volumes until the VM has run for a while), and remove its line from docker-1's `/etc/packetstation/backup.conf`. docker-1 keeps the old `ps-backup.sh` and `ps-restore.sh` it has installed; the versions in the repo are the VM ones and do not run there.
 ## Backups on a VM
 `scripts/backup/ps-backup.sh` (installed as `/usr/local/sbin/ps-backup`) runs from `packetstation-backup.timer`, as root, outside the app, so a broken app still backs up. [scripts/backup/README.md](../scripts/backup/README.md) has the file list.
 - Timer `OnCalendar=*-*-* 02:00:00 America/Denver`, `Persistent=true`: 02:00 Denver time all year, an hour before the 03:00 PBS run, so PBS captures the fresh dumps. A missed run runs at the next boot. `install.sh` enables it.
 - `/etc/packetstation/backup.conf`: one line, `name database folder`, for example `prod packetstation /var/backups/packetstation`. `install.sh` writes it.
 - `pg_dump -d packetstation -Fc` and `pg_dumpall --globals-only --no-role-passwords` run as the `postgres` OS user (`setpriv`, no PAM) on the local socket. No password is read or passed. Root writes the output under a temporary name, `pg_restore --list` must list at least one entry, then the files are renamed to `ps-<UTC yyyymmddThhmmssZ>.dump` and `.globals.sql`, mode 0600.
 - The newest 7 are kept (`KEEP=7` in the unit, `--keep N`), with their globals files. `last-success` or `last-failure` (`<ISO UTC time> <dump name>`, plus the reason on failure), mode 0644, owner and group taken from the folder (root:packetstation 0750). The worker's `backup-check` job (07:15) reads them through `BACKUP_DIR=/var/backups/packetstation`. A failed run leaves no partial file. `flock` keeps two runs apart. Exit 1 on a failure, 2 on a usage or config error.
 - The unit runs with `Nice=10`, `IOSchedulingClass=idle`, `TimeoutStartSec=1h`, `ProtectSystem=strict` (writable only `/var/backups/packetstation`), `ProtectHome`, `PrivateTmp`, `NoNewPrivileges`, and only the capabilities to switch user and set owners.
 - `ps-upgrade` takes a backup before every upgrade.
 What the dump does not hold, and what covers it:
 | What | Where | Covered by |
 |---|---|---|
 | Stored files | `/var/lib/packetstation/assets` | PBS (the VM). Restore with a PBS file-level restore and `ps-restore --assets`. |
 | Settings, secrets, CA | `/etc/packetstation` | PBS. The secrets are also needed to rebuild a VM: without `seal_key` the seals of filed packets cannot be checked. Keep a copy of `/etc/packetstation/secrets` where IT keeps the other service secrets. |
 | Archive copies | the QNAP share | The QNAP's own backup. |
 Check by hand:
 ```bash
 cd ~/packetstation/repo
 sudo install -o root -g root -m 0755 scripts/backup/ps-backup.sh  /usr/local/sbin/ps-backup.sh
 sudo install -o root -g root -m 0755 scripts/backup/ps-restore.sh /usr/local/sbin/ps-restore.sh
 sudo install -d -o root -g root -m 0755 /etc/packetstation
 sudo install -o root -g root -m 0644 scripts/backup/backup.conf.example /etc/packetstation/backup.conf
 sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.service /etc/systemd/system/
 sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.timer   /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemd-analyze verify /etc/systemd/system/packetstation-backup.service
 sudo systemctl enable --now packetstation-backup.timer
 systemctl list-timers packetstation-backup.timer
 sudo systemctl start packetstation-backup.service
 journalctl -u packetstation-backup.service -n 20 --no-pager
-ls -la ~/packetstation/test/backups
+sudo ls -la /var/backups/packetstation
 systemctl list-timers packetstation-backup.timer      # next run 08:00 or 09:00 UTC, 02:00 in Denver
 ```
-`list-timers` shows the next run at 08:00 or 09:00 UTC (02:00 in Denver). The one run by hand should leave a new `ps-*.dump` and a fresh `last-success`. After changing a script in the repo, run its `install` line again: the installed copies are root's, so a sync of the checkout does not change what root runs.
+## Restore on a VM
-When the production stack exists (phase 10):
+`ps-restore` (`scripts/backup/ps-restore.sh`) never stops, starts or reconfigures a service. It restores into a new database next to the live one, and a person decides how to switch. A whole-VM restore from PBS stays the disaster path.
 ```bash
-sudo install -d -o root -g root  -m 0755 /var/backups/packetstation
+B=/var/backups/packetstation
-sudo install -d -o root -g 10001 -m 0750 /var/backups/packetstation/prod
+D=$(sudo ls "$B" | grep '\.dump$' | tail -1)
 sudoedit /etc/packetstation/backup.conf      # uncomment the prod line
 sudo systemctl start packetstation-backup.service
 ```
-A run by hand needs no sudo for the test stack (spencerm is in the docker group):
+# 1. Into a fresh database. Refuses an existing one; --replace drops it first (DROP refuses while anyone is
 #    connected). --replace is refused for packetstation itself.
 sudo ps-restore --dump "$B/$D" --globals "$B/${D%.dump}.globals.sql" --db ps_restored
-```bash
+# 2. Stored files, from a folder or a tar (a PBS file-level restore). Existing files are never overwritten.
-scripts/backup/ps-backup.sh --stack test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
+#    Files end up owned by packetstation. Use a scratch folder for a rehearsal.
-```
+sudo ps-restore --assets /path/to/assets.tar --assets-dir /var/lib/packetstation/assets
-The dumps hold the database only. The assets volume (the files) is not in them. It is covered by PBS, since the volume is part of the VM.
+# 3. Every asset row's sha256 against the files.
-
+sudo ps-restore --verify-assets --db ps_restored --assets-dir /var/lib/packetstation/assets
 ## Restore (phase 9)
 `ps-restore.sh` never stops, restarts or reconfigures a stack. It restores into a new database next to the live one, and a person decides how to switch. A whole-VM restore from PBS stays the disaster path.
 ```bash
 B=/home/spencerm/packetstation/test/backups
 D=$(ls $B/ps-*.dump | tail -1)
 # 1. Into a fresh database. Refuses an existing one; --replace drops it first (DROP refuses while anyone is connected).
 ps-restore.sh --project packetstation-test --dump "$D" --globals "${D%.dump}.globals.sql" --db ps_restored
 # 2. Assets, from a folder or a tar (for example a PBS file-level restore of the assets volume folder).
 #    Existing files are never overwritten. Files end up owned by 10001:10001.
 ps-restore.sh --project packetstation-test --assets /path/to/assets.tar --assets-volume packetstation-test_assets
 # 3. Every asset row's sha256 against the files in the volume.
 ps-restore.sh --project packetstation-test --verify-assets --db ps_restored --assets-volume packetstation-test_assets
 # Row counts of any database, to compare with the live one.
-ps-restore.sh --project packetstation-test --counts-only --db packetstation
+sudo ps-restore --counts-only --db packetstation
 ```
- The database is created as `01-roles.sh` creates it (owner `ps_migrate`, UTF8, `template0`, CONNECT and TEMPORARY for `ps_app`), then `pg_restore --exit-on-error --single-transaction` as postgres. A failed restore leaves an empty database, never a half one.
+- The database is created as `install.sh` creates the live one (owner `ps_migrate`, UTF8, `en_US.UTF-8`, `template0`, CONNECT and TEMPORARY for `ps_app`), then `pg_restore --exit-on-error --single-transaction` as postgres. A failed restore leaves an empty database, never a half one.
- The globals file is applied without stopping on "already exists". Any other error fails. The script refuses a globals file that holds a password hash. On a new cluster the roles come back without passwords: set each from the stack's secret file with `\password` in psql, or start the stack on an empty volume so initdb creates them.
+- The globals file is applied without stopping on "already exists". Any other error fails. A globals file with a password hash is refused. Roles created from it have no password: run `install.sh --skip-packages`, which sets both role passwords from the secrets.
- The assets copy runs in a throwaway container from the image the postgres container already runs, with `--network none`, so nothing is downloaded.
+- To put a restored database into service: `sudo systemctl stop packetstation-app packetstation-worker`; as postgres (`sudo setpriv --reuid=postgres --regid=postgres --init-groups psql -X -d postgres`) run `ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`; then `sudo systemctl restart packetstation-migrate` and `sudo systemctl start packetstation-app packetstation-worker` (migrate brings an older dump up to the release), check the app, and drop the old database. This step is deliberate and manual.
- To put a restored database into service: stop that stack's app and worker, rename as postgres (`ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`), start the stack, check the app, then drop the old database. This step is deliberate and manual.
+- A rebuilt VM after a disaster: copy the saved `/etc/packetstation/secrets` back first, run `install.sh`, `ps-upgrade` with the same release, then the restore and the rename above (the install made an empty `packetstation`), and the assets.
 - The next rehearsal is IT's, quarterly (requirement O2). The first one on the test VM is part of its acceptance.
 ## Backup and restore on docker-1 (phase 9, until the test VM)
 Until the test VM takes over, the docker-1 test stack is backed up by the phase 9 scripts installed on docker-1 as `/usr/local/sbin/ps-backup.sh` and `ps-restore.sh`. They reach the stack's postgres container with `docker exec` and read `/etc/packetstation/backup.conf` in the four-column form `name container database folder` (`test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups`). Those versions are in git history (up to commit 3fdf063). The scripts in the repo are now the VM ones and refuse that config, so do not reinstall from the repo on docker-1. Markers, rotation and timer are the same as on a VM. The rehearsal below was made with them.
 ### Restore rehearsal 2026-10-21
@@ -2,9 +2,7 @@
 ## backup/
-The nightly per-stack `pg_dump` backup with its systemd timer, and the restore script. See
+The nightly `pg_dump` backup with its systemd timer, and the restore script, for the native VMs (no Docker). See [backup/README.md](backup/README.md) and docs/DEPLOY.md, "Backups on a VM" and "Restore on a VM". The VM install kit itself (units, launcher, install and upgrade) is in `deploy/vm/`.
 [backup/README.md](backup/README.md) for what they write and Spencer's install steps. The restore
 rehearsal is recorded in docs/DEPLOY.md.
 ## Export-PescoCaBundle.ps1
@@ -1,99 +1,29 @@
 # Backup and restore
-Host scripts for docker-1. They run outside the app, so a broken app still backs up.
+Scripts for the native VMs (production and test). They run outside the app, so a broken app still backs up. `deploy/vm/install.sh` installs them; the steps and the restore procedure are in [docs/DEPLOY.md](../../docs/DEPLOY.md), "Backups on a VM" and "Restore on a VM".
-| File | What it is |
+| File | Installed as | What it is |
-|---|---|
+|---|---|---|
-| `ps-backup.sh` | Nightly `pg_dump -Fc` of each stack's database plus `pg_dumpall --globals-only --no-role-passwords`, checked with `pg_restore --list`, 7 kept, `last-success` and `last-failure` markers. |
+| `ps-backup.sh` | `/usr/local/sbin/ps-backup` | Nightly `pg_dump -Fc` plus `pg_dumpall --globals-only --no-role-passwords`, as the postgres OS user on the local socket. Checked with `pg_restore --list`, 7 kept, `last-success` and `last-failure` markers. |
-| `ps-restore.sh` | Restores a dump into a fresh database, applies the globals, copies assets into a volume, checks asset hashes, prints row counts. |
+| `ps-restore.sh` | `/usr/local/sbin/ps-restore` | Restores a dump into a fresh database, applies the globals, copies stored files into an assets folder without overwriting, checks every file's sha256 against the database, prints row counts. |
-| `packetstation-backup.service` | systemd oneshot that runs `ps-backup.sh` as root. |
+| `packetstation-backup.service` | `/etc/systemd/system/` | Oneshot as root, sandboxed, writes only `/var/backups/packetstation`. |
-| `packetstation-backup.timer` | 02:00 America/Denver every day, `Persistent=true`. |
+| `packetstation-backup.timer` | `/etc/systemd/system/` | 02:00 America/Denver every day, `Persistent=true`. |
-| `backup.conf.example` | The stack list for `/etc/packetstation/backup.conf`. |
+| `backup.conf.example` | `/etc/packetstation/backup.conf` | `name database folder`, one line per stack. |
-Test: `tests/server/test_backup_scripts.py` runs `ps-backup.sh` with a fake `docker` on PATH.
+Tests: `tests/server/test_backup_scripts.py` runs both scripts with fake PostgreSQL tools (`PG_BINDIR`) and a fake user switch (`PG_AS`).
 ## What a backup is
-In each stack's folder (`/var/backups/packetstation/prod`, `/home/spencerm/packetstation/test/backups`):
+In `/var/backups/packetstation` (root:packetstation 0750):
 - `ps-<UTC stamp>.dump`: the app database, custom format. Mode 0600.
 - `ps-<UTC stamp>.globals.sql`: the roles, without passwords. Mode 0600.
- `last-success`, `last-failure`: one line, `<ISO UTC time> <dump name>` (a failure adds the reason). Mode 0644. The worker's backup-check job compares their modification times.
+- `last-success`, `last-failure`: one line, `<ISO UTC time> <dump name>` (a failure adds the reason). Mode 0644. The worker's backup-check job reads them through the folder's group.
-Each dump is written under a temporary name (`.ps-….partial`), checked, then renamed. A failed run leaves no partial file and writes `last-failure`. The script tries every stack and exits 1 if any failed, 2 for a usage or config error.
+Each dump is written under a temporary name (`.ps-….partial`), checked, then renamed. A failed run leaves no partial file and writes `last-failure`. Exit 1 if any stack failed, 2 for a usage or config error.
-The script connects inside the postgres container as `postgres` over the local socket, which the image trusts. It reads no password.
+Not in the dump: the stored files (`/var/lib/packetstation/assets`), `/etc/packetstation` (env file, secrets, CA) and the archive share. The PBS backup of the VM covers the first two; the archive is on the QNAP.
-Why `--no-role-passwords`: a password hash in a backup file is something to crack offline, and the file also lands in the PBS copy of the VM. The role passwords are in the stack's secrets already. On a fresh volume `deploy/postgres/initdb/01-roles.sh` creates the roles with them. On an existing cluster the roles are already there.
+## The docker-1 test stack
-Ownership: run as root, the new files take the owner and group of the stack's folder, so the test folder stays spencerm's. The worker mounts the folder read-only as uid 10001. It only needs to list the folder and stat the dumps, plus read the markers, so dumps can stay 0600.
+These scripts no longer talk to Docker. The docker-1 test stack keeps the copies installed in `/usr/local/sbin/ps-backup.sh` and `ps-restore.sh` (the phase 9 versions, in git history up to commit 3fdf063) until it is stopped. Do not install the new scripts on docker-1: its `backup.conf` has the old four-column format, which the new script refuses.
 ## Install (Spencer, with sudo, on docker-1)
 Copy the files from the synced checkout in `~/packetstation/repo`. The installed copies are root's, so a later sync of the checkout does not change what root runs.
 ```bash
 cd ~/packetstation/repo
 sudo install -o root -g root -m 0755 scripts/backup/ps-backup.sh  /usr/local/sbin/ps-backup.sh
 sudo install -o root -g root -m 0755 scripts/backup/ps-restore.sh /usr/local/sbin/ps-restore.sh
 sudo install -d -o root -g root -m 0755 /etc/packetstation
 sudo install -o root -g root -m 0644 scripts/backup/backup.conf.example /etc/packetstation/backup.conf
 sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.service /etc/systemd/system/
 sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.timer   /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemd-analyze verify /etc/systemd/system/packetstation-backup.service
 sudo systemctl enable --now packetstation-backup.timer
 systemctl list-timers packetstation-backup.timer       # next run 08:00 or 09:00 UTC, 02:00 in Denver
 sudo systemctl start packetstation-backup.service      # one run now
 journalctl -u packetstation-backup.service -n 20 --no-pager
 ls -la ~/packetstation/test/backups
 ```
 The config lists the test stack only. When the production stack exists (phase 10), make its folder and uncomment its line:
 ```bash
 sudo install -d -o root -g root  -m 0755 /var/backups/packetstation
 sudo install -d -o root -g 10001 -m 0750 /var/backups/packetstation/prod
 sudoedit /etc/packetstation/backup.conf                 # uncomment the prod line
 sudo systemctl start packetstation-backup.service
 ```
 Group 10001 lets the production worker (uid 10001, group 10001) list the folder and read the markers. Its `stack.env` sets `BACKUP_DIR=/var/backups/packetstation/prod`.
 After changing a script in the repo, install it again with the same `install` line.
 ## Run by hand
 As spencerm (in the docker group), against the test stack:
 ```bash
 scripts/backup/ps-backup.sh --stack test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
 ```
 `--stack NAME CONTAINER DATABASE FOLDER` can repeat. CONTAINER is a container name or `project:<compose project>`. `--keep N` (or `KEEP=N`) changes the 7.
 ## Restore
 The script never stops, restarts or reconfigures a stack. Restore into a new database, check it, then decide how to switch.
 ```bash
 B=/home/spencerm/packetstation/test/backups
 D=$(ls $B/ps-*.dump | tail -1)
 # 1. Into a fresh database (refuses an existing one; --replace drops it first, and DROP refuses while anyone is connected).
 ps-restore.sh --project packetstation-test --dump "$D" --globals "${D%.dump}.globals.sql" --db ps_restored
 # 2. Assets: from a folder or a tar (for example a file-level restore out of PBS of the assets volume).
 #    Existing files are never overwritten. Files end up owned by 10001:10001.
 ps-restore.sh --project packetstation-test --assets /path/to/assets.tar --assets-volume packetstation-test_assets
 # 3. Check every asset row's sha256 against the files in the volume.
 ps-restore.sh --project packetstation-test --verify-assets --db ps_restored --assets-volume packetstation-test_assets
 # Row counts of any database, for comparing.
 ps-restore.sh --project packetstation-test --counts-only --db packetstation
 ```
 To put the restored database into service, stop the app and worker of that stack, rename the databases as `postgres` (`ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`), start the stack again, and drop the old one once the app is checked. That step is deliberate and manual.
 The globals file holds no passwords. If a role had to be created from it on a new cluster, set its password from the stack's secret file with `\password` in psql, or start the stack on an empty volume so initdb creates the roles.
@@ -1,5 +1,4 @@
-# /etc/packetstation/backup.conf: the stacks ps-backup.sh backs up, one per line.
+# /etc/packetstation/backup.conf: the stacks ps-backup backs up, one per line.
-# name   container (or project:<compose project>)   database       folder
+# deploy/vm/install.sh writes the VM's own line when the file is missing. A VM has one stack.
-test     project:packetstation-test                  packetstation  /home/spencerm/packetstation/test/backups
+# name   database       folder
-# Uncomment once the production stack runs (phase 10). Until then it would fail every night.
+prod     packetstation  /var/backups/packetstation
 #prod    project:packetstation                       packetstation  /var/backups/packetstation/prod
@@ -1,25 +1,29 @@
-# Packet Station nightly database backup (scripts/backup/ps-backup.sh).
+# Packet Station nightly database backup on a native VM (scripts/backup/ps-backup.sh, installed as
-# Installed by root on docker-1: see scripts/backup/README.md. Started by packetstation-backup.timer.
+# /usr/local/sbin/ps-backup by deploy/vm/install.sh). Started by packetstation-backup.timer.
-# Runs as root because it talks to the Docker daemon and writes /var/backups/packetstation.
+# Runs as root: it reads the backup config, writes /var/backups/packetstation and switches to the
 # postgres OS user (setpriv) for pg_dump on the local socket.
 [Unit]
-Description=Packet Station database backup (pg_dump of each stack)
+Description=Packet Station database backup (pg_dump)
-Documentation=file:///home/spencerm/packetstation/repo/scripts/backup/README.md
+Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
-Wants=docker.service
+Wants=[email protected]
-After=docker.service
+After=[email protected]
 [Service]
 Type=oneshot
-ExecStart=/usr/local/sbin/ps-backup.sh --config /etc/packetstation/backup.conf
+ExecStart=/usr/local/sbin/ps-backup --config /etc/packetstation/backup.conf
 Environment=KEEP=7
-# Gentle on the host: the dump runs while the shop is closed, but the stacks stay up.
+# Gentle on the VM: the dump runs while the shop is closed, but the app stays up.
 Nice=10
 IOSchedulingClass=idle
 TimeoutStartSec=1h
-# Hardening. /usr, /boot and /etc are read-only to the script; it writes only the backup folders.
+# Sandbox. Read-only file system except the backup folder. setpriv needs CAP_SETUID and CAP_SETGID.
 NoNewPrivileges=yes
 PrivateTmp=yes
-ProtectSystem=full
+ProtectSystem=strict
 ProtectHome=yes
 ReadWritePaths=/var/backups/packetstation
 ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectControlGroups=yes
 RestrictSUIDSGID=yes
 CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_CHOWN CAP_FOWNER CAP_DAC_OVERRIDE
@@ -1,5 +1,5 @@
 # Packet Station nightly backup at 02:00 Denver time, an hour before the 03:00 Proxmox Backup
-# Server run, so PBS captures the fresh dumps. The host clock is UTC; the zone makes the intent
+# Server run, so PBS captures the fresh dumps. The VM clock may be UTC; the zone makes the intent
 # explicit and follows daylight saving. Persistent=true runs a missed backup at the next boot.
 [Unit]
 Description=Packet Station nightly database backup
@@ -1,50 +1,52 @@
 #!/usr/bin/env bash
-# Packet Station nightly backup. Runs on the docker host, never inside the app.
+# Packet Station nightly backup on a native VM. Runs as root from packetstation-backup.timer,
 # outside the app, so a broken app still backs up.
 #
 # For each configured stack it writes, into that stack's backup folder:
 #   ps-<UTC stamp>.dump          pg_dump -Fc of the app database (custom format)
 #   ps-<UTC stamp>.globals.sql   pg_dumpall --globals-only --no-role-passwords (roles, no password hashes)
 #   last-success / last-failure  one line: "<ISO UTC time> <dump name>" (last-failure adds the reason)
 #
-# Each dump is written under a temporary name, checked with `pg_restore --list` run inside the
+# Each dump is written under a temporary name, checked with `pg_restore --list`, and only then
-# postgres container, and only then renamed into place. The newest KEEP dumps (default 7) are kept,
+# renamed into place. The newest KEEP dumps (default 7) are kept, with their globals files. Any
-# with their globals files. Any failure writes last-failure and the script exits non-zero after
+# failure writes last-failure and the script exits non-zero after trying the remaining stacks.
 # trying the remaining stacks.
 #
 # Usage:
-#   ps-backup.sh [--config FILE] [STACK ...]
+#   ps-backup [--config FILE] [--keep N] [STACK ...]
-#   ps-backup.sh --stack NAME CONTAINER DATABASE FOLDER [--stack ...]
+#   ps-backup --stack NAME DATABASE FOLDER [--stack ...] [--keep N]
 #
 # Config file (default /etc/packetstation/backup.conf, or PS_BACKUP_CONFIG), one stack per line:
-#   # name   container                    database       folder
+#   # name   database       folder
-#   prod     project:packetstation        packetstation  /var/backups/packetstation/prod
+#   prod     packetstation  /var/backups/packetstation
-#   test     project:packetstation-test   packetstation  /home/spencerm/packetstation/test/backups
+# With STACK names given, only those stacks run. A VM has one stack; install.sh writes its line.
 # CONTAINER is a container name, or project:<compose project> to find that project's postgres
 # service by its compose labels. With STACK names given, only those stacks run.
 #
-# pg_dump connects inside the container as the postgres superuser over the local socket, which
+# pg_dump and pg_dumpall run as the postgres OS user (setpriv, no PAM) on the local socket, where
-# the official image trusts. No password is read, passed or printed.
+# PostgreSQL knows that user by peer authentication. No password is read, passed or printed. Their
 # output goes to files root opened, so the postgres user needs no access to the folder.
 # PG_BINDIR (default /usr/lib/postgresql/16/bin) picks the client tools, PG_AS the user switch.
 #
 # Globals: --no-role-passwords, so no password hash ever lands in a file (or in the PBS copy of
-# it). The roles' passwords come from the stack's secrets: initdb (deploy/postgres/initdb)
+# it). The role passwords are in /etc/packetstation/secrets; deploy/vm/install.sh sets them.
 # recreates them on a fresh volume. The globals file is also 0600, like the dumps.
 #
 # Files: dumps and globals are 0600. The markers are 0644 (a time and a file name, nothing else)
-# so the worker, which mounts the folder read-only as uid 10001, can read them. When run as root
+# so the worker, which reaches the folder through its group, can read them. When run as root the
-# the new files take the folder's owner and group, so a stack folder owned by spencerm stays
+# new files take the folder's owner and group.
-# readable by spencerm.
+#
 # Not in the dump: the stored files (/var/lib/packetstation/assets), the env file and the secrets
 # (/etc/packetstation). The PBS backup of the VM covers them. See docs/DEPLOY.md.
 set -euo pipefail
 umask 077
 KEEP="${KEEP:-7}"
-PG_USER="${PG_USER:-postgres}"
+PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
-DOCKER="${DOCKER:-docker}"
+# The command prefix that runs a client tool as the postgres OS user, split on spaces.
 read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"
 log() { printf '%s ps-backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
 usage() {
-  sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' >&2
+  sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' >&2
  exit 2
 }
@@ -53,7 +55,7 @@ usage() {
 config="${PS_BACKUP_CONFIG:-/etc/packetstation/backup.conf}"
 config_given=0
 declare -a only=()
-declare -a stacks=()   # each entry: "name|container|database|folder"
+declare -a stacks=()   # each entry: "name|database|folder"
 while [[ $# -gt 0 ]]; do
  case "$1" in
@@ -61,8 +63,8 @@ while [[ $# -gt 0 ]]; do
      [[ $# -ge 2 ]] || usage
      config="$2"; config_given=1; shift 2 ;;
    --stack)
-      [[ $# -ge 5 ]] || usage
+      [[ $# -ge 4 ]] || usage
-      stacks+=("$2|$3|$4|$5"); shift 5 ;;
+      stacks+=("$2|$3|$4"); shift 4 ;;
    --keep)
      [[ $# -ge 2 ]] || usage
      KEEP="$2"; shift 2 ;;
@@ -82,13 +84,13 @@ if [[ ${#stacks[@]} -eq 0 ]]; then
    log "no --stack given and the config $config is missing or unreadable"
    exit 2
  fi
-  while read -r name container database folder extra; do
+  while read -r name database folder extra; do
    [[ -z "${name:-}" || "${name:0:1}" == "#" ]] && continue
    if [[ -z "${folder:-}" || -n "${extra:-}" ]]; then
-      log "config line for '$name' needs exactly: name container database folder"
+      log "config line for '$name' needs exactly: name database folder"
      exit 2
    fi
-    stacks+=("$name|$container|$database|$folder")
+    stacks+=("$name|$database|$folder")
  done < "$config"
 elif [[ $config_given -eq 1 ]]; then
  log "use --config or --stack, not both"
@@ -102,26 +104,9 @@ fi
 # ------------------------------------------------------------------ helpers
-# Resolve "project:<name>" to the running postgres container of that compose project.
+# A PostgreSQL client tool: from PG_BINDIR when it is set, else from PATH.
-resolve_container() {
+pgbin() {
-  local spec="$1"
+  if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
  if [[ "$spec" == project:* ]]; then
    local project="${spec#project:}" ids
    ids="$("$DOCKER" ps -q \
      --filter "label=com.docker.compose.project=$project" \
      --filter "label=com.docker.compose.service=postgres")"
    if [[ -z "$ids" ]]; then
      echo "no running postgres container for compose project $project" >&2
      return 1
    fi
    if [[ "$(printf '%s\n' "$ids" | wc -l)" -ne 1 ]]; then
      echo "more than one postgres container for compose project $project" >&2
      return 1
    fi
    printf '%s\n' "$ids"
  else
    printf '%s\n' "$spec"
  fi
 }
 # Write a marker atomically: temp file in the same folder, then rename.
@@ -203,8 +188,8 @@ on_stack_exit() {
 }
 backup_stack() {
-  local name="$1" spec="$2" database="$3" folder="$4"
+  local name="$1" database="$2" folder="$3"
-  local stamp globals_name container started entries size
+  local stamp globals_name started entries size
  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
  cur_dump_name="ps-$stamp.dump"
@@ -213,34 +198,33 @@ backup_stack() {
  log "stack $name: database $database into $folder"
  if [[ ! -d "$folder" ]]; then
-    # 0750: the worker reaches the folder through its group (see scripts/backup/README.md).
+    # 0750: the worker reaches the folder through its group (docs/DEPLOY.md, "Backups on a VM").
    mkdir -p -- "$folder"
    chmod 0750 -- "$folder" 2>/dev/null || log "stack $name: could not set mode 0750 on $folder"
  fi
  cur_folder="$folder"
  [[ -w "$folder" ]] || fail_stack "$folder is not writable"
  [[ "$database" =~ ^[A-Za-z_][A-Za-z0-9_]{0,62}$ ]] || fail_stack "database name '$database' is not a plain name"
  if command -v flock >/dev/null 2>&1; then
    exec 9>"$folder/.ps-backup.lock"
    flock -n 9 || fail_stack "another backup of this stack is running"
  fi
  container="$(resolve_container "$spec" 2>&1)" || fail_stack "$container"
  cur_tmp_dump="$(mktemp "$folder/.$cur_dump_name.XXXXXX.partial")"
  cur_tmp_globals="$(mktemp "$folder/.$globals_name.XXXXXX.partial")"
-  "$DOCKER" exec "$container" pg_dump -U "$PG_USER" -d "$database" -Fc > "$cur_tmp_dump" \
+  "${PG_AS[@]}" "$(pgbin pg_dump)" -d "$database" -Fc > "$cur_tmp_dump" \
    || fail_stack "pg_dump failed"
  [[ -s "$cur_tmp_dump" ]] || fail_stack "pg_dump wrote nothing"
-  "$DOCKER" exec "$container" pg_dumpall -U "$PG_USER" --globals-only --no-role-passwords \
+  "${PG_AS[@]}" "$(pgbin pg_dumpall)" --globals-only --no-role-passwords \
    > "$cur_tmp_globals" || fail_stack "pg_dumpall --globals-only failed"
  [[ -s "$cur_tmp_globals" ]] || fail_stack "pg_dumpall wrote nothing"
-  # The check reads the dump from stdin with the container's own pg_restore. Entry lines start
+  # The check reads the dump from stdin and needs no database. Entry lines start with the
-  # with the entry's id; comment lines start with a semicolon.
+  # entry's id; comment lines start with a semicolon.
-  entries="$("$DOCKER" exec -i "$container" pg_restore --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
+  entries="$("$(pgbin pg_restore)" --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
    || fail_stack "pg_restore --list could not read the dump"
  [[ "$entries" -ge 1 ]] || fail_stack "pg_restore --list found no entries"
@@ -269,7 +253,7 @@ run_stack() (
 failures=0
 ran=0
 for entry in "${stacks[@]}"; do
-  IFS='|' read -r s_name s_container s_database s_folder <<< "$entry"
+  IFS='|' read -r s_name s_database s_folder <<< "$entry"
  if [[ ${#only[@]} -gt 0 ]]; then
    wanted=0
    for o in "${only[@]}"; do
@@ -280,7 +264,7 @@ for entry in "${stacks[@]}"; do
  ran=$(( ran + 1 ))
  # Not in an if or ||: that would switch errexit off inside the subshell.
  set +e
-  run_stack "$s_name" "$s_container" "$s_database" "$s_folder"
+  run_stack "$s_name" "$s_database" "$s_folder"
  rc=$?
  set -e
  if [[ $rc -ne 0 ]]; then
@@ -1,75 +1,73 @@
 #!/usr/bin/env bash
-# Packet Station restore. Runs on the docker host. Restores a ps-backup.sh dump into a fresh
+# Packet Station restore on a native VM. Run by root. Restores a ps-backup dump into a fresh
-# database on a stack's postgres container, optionally applies the globals file, copies assets
+# database, optionally applies the globals file, copies stored files back into an assets folder,
-# back into an assets volume, and prints the row count of every table at the end.
+# checks them against the database, and prints the row count of every table at the end.
 #
-# It never stops, restarts or reconfigures a stack. Pointing the app at the restored database
+# It never stops, starts or reconfigures a service. Putting the restored database into service
-# (or renaming it into place) is a separate, deliberate step: see scripts/backup/README.md.
+# (renaming it into place) is a separate, deliberate step: see docs/DEPLOY.md, "Restore on a VM".
 #
 # Usage:
-#   ps-restore.sh --container NAME | --project PROJECT
+#   ps-restore [--dump FILE --db NAME [--replace]]
 #                 [--dump FILE --db NAME [--replace]]
 #              [--globals FILE]
-#                 [--assets DIR_OR_TAR --assets-volume VOLUME]
+#              [--assets DIR_OR_TAR --assets-dir DIR]
-#                 [--verify-assets --db NAME --assets-volume VOLUME]
+#              [--verify-assets --db NAME --assets-dir DIR]
 #              [--counts-only --db NAME]
 #
 #   --container / --project  the postgres container, by name or by compose project.
 #   --dump FILE              a ps-*.dump file (pg_dump -Fc).
 #   --db NAME                the database to restore into. It must not exist, unless --replace,
 #                            which drops it first. DROP DATABASE refuses while anyone is connected.
-#   --owner ROLE             owner of the new database (default ps_migrate, as initdb makes it).
+#                            --replace is refused for packetstation, the live database.
 #   --owner ROLE             owner of the new database (default ps_migrate, as install.sh makes it).
 #   --app-role ROLE          role given CONNECT and TEMPORARY on it (default ps_app).
 #   --locale NAME            the new database's locale (default en_US.UTF-8, as install.sh makes it).
 #   --globals FILE           a ps-*.globals.sql file. Roles that already exist are reported and
-#                            left alone. The file holds no passwords: set them from the secrets.
+#                            left alone. The file holds no passwords: install.sh sets them again.
 #   --assets SRC             a folder, or a .tar / .tar.gz / .tgz, holding the files that belong
-#                            under ASSETS_DIR. Copied into --assets-volume through a throwaway
+#                            under ASSETS_DIR. Existing files are never overwritten.
-#                            container (no network) that runs the postgres image already on the host.
+#   --assets-dir DIR         the folder to copy into or check (/var/lib/packetstation/assets is the
-#                            Existing files are never overwritten.
+#                            live one; a scratch folder for a rehearsal).
-#   --assets-volume VOLUME   the docker volume to copy into (for example packetstation-test_assets).
+#   --assets-owner U:G       owner for the copied files (default packetstation:packetstation).
-#   --assets-owner UID:GID   owner for the copied files (default 10001:10001, the image's ps user).
+#   --verify-assets          compare sha256 of every file in --assets-dir with assets.sha256 in --db.
 #   --verify-assets          compare sha256 of every file in --assets-volume with assets.sha256 in --db.
 #   --counts-only            only print the row counts of --db.
 #
-# Connects inside the container as the postgres superuser over the local socket. No password
+# psql and pg_restore run as the postgres OS user on the local socket (peer authentication). The
-# is read, passed or printed.
+# files are opened by root and handed over on stdin. No password is read, passed or printed.
 set -euo pipefail
 umask 077
-PG_USER="${PG_USER:-postgres}"
+PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
-DOCKER="${DOCKER:-docker}"
+read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"
 LIVE_DB=packetstation
 log() { printf '%s ps-restore: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
 die() { log "$*"; exit 1; }
-usage() { sed -n '2,36p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+usage() { sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
 container=""
 project=""
 dump=""
 db=""
 replace=0
 owner="ps_migrate"
 app_role="ps_app"
 locale_name="en_US.UTF-8"
 globals=""
 assets=""
-assets_volume=""
+assets_dir=""
-assets_owner="10001:10001"
+assets_owner="packetstation:packetstation"
 verify_assets=0
 counts_only=0
 while [[ $# -gt 0 ]]; do
  case "$1" in
    --container) container="${2:?}"; shift 2 ;;
    --project) project="${2:?}"; shift 2 ;;
    --dump) dump="${2:?}"; shift 2 ;;
    --db) db="${2:?}"; shift 2 ;;
    --replace) replace=1; shift ;;
    --owner) owner="${2:?}"; shift 2 ;;
    --app-role) app_role="${2:?}"; shift 2 ;;
    --locale) locale_name="${2:?}"; shift 2 ;;
    --globals) globals="${2:?}"; shift 2 ;;
    --assets) assets="${2:?}"; shift 2 ;;
-    --assets-volume) assets_volume="${2:?}"; shift 2 ;;
+    --assets-dir) assets_dir="${2:?}"; shift 2 ;;
    --assets-owner) assets_owner="${2:?}"; shift 2 ;;
    --verify-assets) verify_assets=1; shift ;;
    --counts-only) counts_only=1; shift ;;
@@ -82,32 +80,31 @@ ident_re='^[a-z_][a-z0-9_]{0,62}$'
 [[ -z "$db" || "$db" =~ $ident_re ]] || die "--db must be a plain lower-case name (letters, digits, _)"
 [[ "$owner" =~ $ident_re ]] || die "--owner must be a plain lower-case role name"
 [[ "$app_role" =~ $ident_re ]] || die "--app-role must be a plain lower-case role name"
-[[ "$assets_owner" =~ ^[0-9]+:[0-9]+$ ]] || die "--assets-owner must be UID:GID"
+[[ "$locale_name" =~ ^[A-Za-z0-9_.@-]+$ ]] || die "--locale must be a locale name like en_US.UTF-8"
-
+[[ "$assets_owner" =~ ^[a-z_0-9][a-z0-9_-]*:[a-z_0-9][a-z0-9_-]*$ ]] || die "--assets-owner must be USER:GROUP"
 if [[ -n "$project" ]]; then
  [[ -z "$container" ]] || die "use --container or --project, not both"
  container="$("$DOCKER" ps -q \
    --filter "label=com.docker.compose.project=$project" \
    --filter "label=com.docker.compose.service=postgres")"
  [[ -n "$container" ]] || die "no running postgres container for compose project $project"
  [[ "$(printf '%s\n' "$container" | wc -l)" -eq 1 ]] || die "more than one postgres container for $project"
 fi
 [[ -n "$container" ]] || die "--container or --project is required"
 if [[ -n "$dump" || $counts_only -eq 1 || $verify_assets -eq 1 ]]; then
  [[ -n "$db" ]] || die "--db is required"
 fi
 if [[ -n "$assets" || $verify_assets -eq 1 ]]; then
-  [[ -n "$assets_volume" ]] || die "--assets-volume is required"
+  [[ -n "$assets_dir" ]] || die "--assets-dir is required"
  [[ -d "$assets_dir" ]] || die "$assets_dir is not a folder"
 fi
 if [[ -z "$dump" && -z "$globals" && -z "$assets" && $verify_assets -eq 0 && $counts_only -eq 0 ]]; then
  die "nothing to do: give --dump, --globals, --assets, --verify-assets or --counts-only"
 fi
 if [[ -n "$dump" && $replace -eq 1 && "$db" == "$LIVE_DB" ]]; then
  die "--replace is refused for $LIVE_DB. Restore into another name and rename it into place (docs/DEPLOY.md)."
 fi
-# psql inside the container, reading SQL from stdin. Variables go through -v and :'name'.
+pgbin() {
  if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
 }
 # psql as postgres, reading SQL from stdin. Variables go through -v and :'name'.
 psql_c() {
  local dbname="$1"; shift
-  "$DOCKER" exec -i "$container" psql -X -q -v ON_ERROR_STOP=1 -U "$PG_USER" -d "$dbname" "$@"
+  "${PG_AS[@]}" "$(pgbin psql)" -X -q -v ON_ERROR_STOP=1 -d "$dbname" "$@"
 }
 db_exists() {
@@ -143,22 +140,16 @@ ORDER BY c.relname;
 SQL
 }
 # A throwaway container on no network, running the image the postgres container already uses,
 # so nothing is downloaded.
 helper_image() {
  "$DOCKER" inspect -f '{{.Config.Image}}' "$container"
 }
 # ------------------------------------------------------------------ globals
 if [[ -n "$globals" ]]; then
  [[ -r "$globals" ]] || die "cannot read $globals"
  if grep -qiE "PASSWORD[[:space:]]+'(SCRAM|md5)" "$globals"; then
-    die "$globals holds password hashes; ps-backup.sh writes globals without them. Refusing."
+    die "$globals holds password hashes; ps-backup writes globals without them. Refusing."
  fi
  log "applying globals from $(basename -- "$globals") (existing roles are left as they are)"
  # Without ON_ERROR_STOP: CREATE ROLE for a role that exists fails and the rest goes on.
-  out="$("$DOCKER" exec -i "$container" psql -X -q -U "$PG_USER" -d postgres < "$globals" 2>&1 || true)"
+  out="$("${PG_AS[@]}" "$(pgbin psql)" -X -q -d postgres < "$globals" 2>&1 || true)"
  exists="$(printf '%s\n' "$out" | grep -c 'already exists' || true)"
  other="$(printf '%s\n' "$out" | grep 'ERROR' | grep -vc 'already exists' || true)"
  log "globals: $exists statements skipped (already exists), $other other errors"
@@ -180,12 +171,13 @@ if [[ -n "$dump" ]]; then
 SELECT format('DROP DATABASE %I', :'name') \gexec
 SQL
  fi
-  role_exists "$owner" || die "role $owner does not exist (apply --globals or start the stack once)"
+  role_exists "$owner" || die "role $owner does not exist (apply --globals, or run deploy/vm/install.sh)"
  log "creating database $db owned by $owner"
-  # The same shape as deploy/postgres/initdb/01-roles.sh.
+  # The same shape as deploy/vm/install.sh makes the live one.
-  psql_c postgres -v name="$db" -v owner="$owner" <<'SQL'
+  psql_c postgres -v name="$db" -v owner="$owner" -v loc="$locale_name" <<'SQL'
-SELECT format('CREATE DATABASE %I OWNER %I ENCODING %L TEMPLATE template0', :'name', :'owner', 'UTF8') \gexec
+SELECT format('CREATE DATABASE %I OWNER %I ENCODING %L LOCALE %L TEMPLATE template0',
              :'name', :'owner', 'UTF8', :'loc') \gexec
 SELECT format('REVOKE ALL ON DATABASE %I FROM PUBLIC', :'name') \gexec
 SQL
  if role_exists "$app_role"; then
@@ -198,8 +190,8 @@ SQL
  log "restoring $(basename -- "$dump") ($(stat -c %s "$dump") bytes) into $db"
  # One transaction: a restore that fails leaves an empty database, never a half-restored one.
-  "$DOCKER" exec -i "$container" pg_restore -U "$PG_USER" -d "$db" --exit-on-error --single-transaction \
+  "${PG_AS[@]}" "$(pgbin pg_restore)" -d "$db" --exit-on-error --single-transaction < "$dump" \
-    < "$dump" || die "pg_restore failed; database $db is left empty. Drop it with --replace on the next try."
+    || die "pg_restore failed; database $db is left empty. Drop it with --replace on the next try."
  log "database restored in $(( SECONDS - started )) s"
 fi
@@ -207,26 +199,21 @@ fi
 if [[ -n "$assets" ]]; then
  [[ -e "$assets" ]] || die "$assets does not exist"
  "$DOCKER" volume inspect "$assets_volume" >/dev/null 2>&1 || die "volume $assets_volume does not exist"
  image="$(helper_image)"
  started=$SECONDS
  src="$(cd -- "$(dirname -- "$assets")" && pwd)/$(basename -- "$assets")"
  if [[ -d "$assets" ]]; then
-    log "copying assets from folder $src into volume $assets_volume"
+    log "copying assets from folder $assets into $assets_dir"
-    "$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
+    cp -a -n -- "$assets/." "$assets_dir/"
      -v "$assets_volume:/dst" -v "$src:/src:ro" "$image" -c \
      'set -eu; cp -a -n /src/. /dst/; chown -R "$1" /dst' sh "$assets_owner"
  else
    case "$assets" in
      *.tar.gz|*.tgz) tar_flags="-xzf" ;;
      *.tar) tar_flags="-xf" ;;
      *) die "--assets must be a folder, .tar, .tar.gz or .tgz" ;;
    esac
-    log "copying assets from $(basename -- "$src") into volume $assets_volume"
+    log "copying assets from $(basename -- "$assets") into $assets_dir"
-    "$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
+    tar "$tar_flags" "$assets" -C "$assets_dir" --skip-old-files --no-same-owner
-      -v "$assets_volume:/dst" -v "$src:/src.tar:ro" "$image" -c \
+  fi
-      'set -eu; tar "$1" /src.tar -C /dst --skip-old-files --no-same-owner; chown -R "$2" /dst' \
+  if [[ "$(id -u)" == "0" ]]; then
-      sh "$tar_flags" "$assets_owner"
+    chown -R -- "$assets_owner" "$assets_dir"
  fi
  log "assets copied in $(( SECONDS - started )) s"
 fi
@@ -234,8 +221,6 @@ fi
 # ------------------------------------------------------------------ verify assets
 if [[ $verify_assets -eq 1 ]]; then
  "$DOCKER" volume inspect "$assets_volume" >/dev/null 2>&1 || die "volume $assets_volume does not exist"
  image="$(helper_image)"
  work="$(mktemp -d)"
  trap 'rm -rf -- "$work"' EXIT
  # sha256sum check format: "<hex>  <path>".
@@ -243,21 +228,11 @@ if [[ $verify_assets -eq 1 ]]; then
 SELECT encode(sha256, 'hex') || '  ' || storage_key FROM assets ORDER BY storage_key;
 SQL
  rows="$(wc -l < "$work/assets.sha256")"
-  chmod 0755 "$work"
+  log "checking $rows asset rows of $db against $assets_dir"
-  chmod 0644 "$work/assets.sha256"
+  ( cd -- "$assets_dir" && sha256sum -c "$work/assets.sha256" 2>/dev/null ) > "$work/result" || true
-  log "checking $rows asset rows of $db against volume $assets_volume"
+  ( cd -- "$assets_dir" && find . -type f ! -path './tmp/*' | sed 's|^\./||' | LC_ALL=C sort ) > "$work/files"
-  set +e
+  cut -c67- "$work/assets.sha256" | LC_ALL=C sort > "$work/rows"
-  "$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
+  LC_ALL=C comm -23 "$work/files" "$work/rows" | sed 's|^|ORPHAN: |' >> "$work/result"
    -v "$assets_volume:/dst:ro" -v "$work:/work:ro" "$image" -c '
      cd /dst || exit 3
      sha256sum -c /work/assets.sha256 2>/dev/null
      find . -type f ! -path "./tmp/*" | sed "s|^\./||" | sort > /tmp/files
      cut -c67- /work/assets.sha256 | sort > /tmp/rows
      comm -23 /tmp/files /tmp/rows | sed "s|^|ORPHAN: |"
      exit 0' > "$work/result"
  rc=$?
  set -e
  [[ $rc -eq 0 ]] || die "the check container failed (exit $rc)"
  ok="$(grep -c ': OK$' "$work/result" || true)"
  bad="$(grep -c ': FAILED$' "$work/result" || true)"
  missing="$(grep -c ': FAILED open or read$' "$work/result" || true)"
@@ -1,12 +1,14 @@
-"""scripts/backup/ps-backup.sh with a fake `docker` on PATH.
+"""The VM backup and restore scripts (scripts/backup) and the VM kit's shell parts (deploy/vm).
-No Docker and no database: the fake answers `docker ps`, `docker exec ... pg_dump`, `pg_dumpall` and
+No PostgreSQL, no systemd and no root: fake `pg_dump`, `pg_dumpall`, `pg_restore` and `psql` sit in a
-`pg_restore --list`, and records every call. Runs on Linux (CI, docker-1) and on Windows when a
+folder given as PG_BINDIR, a fake user switch stands in for setpriv (PG_AS), and a fake `systemd-run`
-Git Bash or Cygwin bash with cygpath is installed; skipped otherwise.
+prints what ps-run would start. Every fake records its calls. Runs on Linux (CI, docker-1) and on
 Windows when a Git Bash or Cygwin bash with cygpath is installed; skipped otherwise.
 """
 from __future__ import annotations
 import hashlib
 import os
 import re
 import shutil
@@ -20,29 +22,23 @@ import pytest
 REPO = Path(__file__).resolve().parents[2]
 BACKUP = REPO / "scripts" / "backup" / "ps-backup.sh"
 RESTORE = REPO / "scripts" / "backup" / "ps-restore.sh"
 VM = REPO / "deploy" / "vm"
 UNITS = VM / "systemd"
 WINDOWS = sys.platform == "win32"
-FAKE_DOCKER = r"""#!/usr/bin/env bash
+# One script, written under each tool's name. It dispatches on its own name.
-# Fake docker for the backup script tests. Behaviour comes from FAKE_* variables.
+FAKE_PG = r"""#!/usr/bin/env bash
-printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG"
+tool="$(basename "$0")"
-cmd="$1"; shift
+printf '%s %s\n' "$tool" "$*" >> "$FAKE_LOG"
-case "$cmd" in
+case "$tool" in
-  ps)
+  asuser)
-    if [[ -n "${FAKE_PS_IDS+x}" ]]; then
+    exec "$@" ;;
      [[ -n "$FAKE_PS_IDS" ]] && printf '%s\n' $FAKE_PS_IDS
    else
      echo pgfake
    fi
    exit 0 ;;
  exec)
    [[ "$1" == "-i" ]] && shift
    container="$1"; shift
    tool="$1"; shift
    case "$tool" in
  pg_dump)
-        [[ "$container" == "${FAKE_DUMP_FAIL_CONTAINER:-none}" ]] && { echo "pg_dump: error: fake" >&2; exit 1; }
+    db=""
    while [[ $# -gt 0 ]]; do [[ "$1" == "-d" ]] && db="$2"; shift; done
    [[ "$db" == "${FAKE_DUMP_FAIL_DB:-none}" ]] && { echo "pg_dump: error: fake" >&2; exit 1; }
    [[ -n "${FAKE_DUMP_EMPTY:-}" ]] && exit 0
-        printf 'PGDMP fake dump of %s\n' "$container"
+    printf 'PGDMP fake dump of %s\n' "$db"
    exit 0 ;;
  pg_dumpall)
    printf 'CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN;\n'
@@ -51,15 +47,40 @@ case "$cmd" in
    input="$(cat)"
    [[ -n "${FAKE_RESTORE_FAIL:-}" ]] && { echo "pg_restore: error: input file is too short" >&2; exit 1; }
    [[ "$input" == PGDMP* ]] || { echo "pg_restore: error: not a dump" >&2; exit 1; }
    if [[ " $* " == *" --list "* ]]; then
      printf ';\n; Archive created at 2026-10-21\n;\n215; 1259 16390 TABLE public packets ps_migrate\n'
      printf '3801; 0 16390 TABLE DATA public packets ps_migrate\n'
    fi
    exit 0 ;;
  psql)
    sql="$(cat)"
    printf '%s\n' "$sql" | sed 's/^/  sql: /' >> "$FAKE_LOG"
    case "$sql" in
      *"FROM pg_database"*) echo "${FAKE_DB_EXISTS:-0}" ;;
      *"FROM pg_roles"*) echo "${FAKE_ROLE_EXISTS:-1}" ;;
      *"FROM assets"*) cat "$FAKE_ASSET_ROWS" ;;
      *"relkind"*) printf 'packets\t3\n' ;;
      *"CREATE ROLE"*) printf '%s\n' "${FAKE_GLOBALS_ERR:-}" >&2 ;;
    esac
    exit 0 ;;
    esac ;;
 esac
-echo "fake docker: unexpected call: $cmd $*" >&2
+echo "fake: unexpected tool $tool" >&2
 exit 99
 """
 FAKE_SYSTEMD_RUN = r"""#!/usr/bin/env bash
 for a in "$@"; do printf '%s\n' "$a"; done
 """
 FAKE_PACKETSTATION = r"""#!/usr/bin/env bash
 printf 'args=%s\n' "$*"
 for v in PG_APP_PASSWORD_FILE PG_MIGRATE_PASSWORD_FILE VISUAL_PASSWORD_FILE MFILES_PASSWORD_FILE \
         SESSION_KEY_FILE SEAL_KEY_FILE STATIC_DIR PS_BUILD WORKER_BEAT_FILE TZ; do
  printf '%s=%s\n' "$v" "${!v-<unset>}"
 done
 printf 'MFILES_PASSWORD=%s\n' "${MFILES_PASSWORD-<unset>}"
 """
 def _find_bash() -> str | None:
    if not WINDOWS:
@@ -74,7 +95,7 @@ def _find_bash() -> str | None:
            continue
        try:
            out = subprocess.run(
-                [c, "-c", "cygpath -u 'C:\\' && command -v find stat mktemp"],
+                [c, "-c", "cygpath -u 'C:\\' && command -v find stat mktemp sha256sum"],
                capture_output=True,
                text=True,
                timeout=30,
@@ -87,7 +108,7 @@ def _find_bash() -> str | None:
 BASH = _find_bash()
-pytestmark = pytest.mark.skipif(BASH is None, reason="no usable bash (Linux bash, or Git Bash with cygpath)")
+needs_bash = pytest.mark.skipif(BASH is None, reason="no usable bash (Linux bash, or Git Bash with cygpath)")
 def _posix(p: Path) -> str:
@@ -98,33 +119,45 @@ def _posix(p: Path) -> str:
    return out.stdout.strip()
 def _write_exec(path: Path, text: str) -> None:
    path.write_text(text, encoding="utf-8", newline="\n")
    path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@pytest.fixture()
 def fake(tmp_path: Path) -> dict[str, object]:
-    bindir = tmp_path / "bin"
+    bindir = tmp_path / "pgbin"
    bindir.mkdir()
-    docker = bindir / "docker"
+    for tool in ("asuser", "pg_dump", "pg_dumpall", "pg_restore", "psql"):
-    docker.write_text(FAKE_DOCKER, encoding="utf-8", newline="\n")
+        _write_exec(bindir / tool, FAKE_PG)
-    docker.chmod(docker.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    log = tmp_path / "calls.log"
    log = tmp_path / "docker.log"
    log.touch()
    env = {k: v for k, v in os.environ.items() if not k.startswith("FAKE_")}
-    env["FAKE_DOCKER_LOG"] = _posix(log)
+    env["FAKE_LOG"] = _posix(log)
-    env["DOCKER"] = _posix(docker)
+    env["PG_BINDIR"] = _posix(bindir)
-    env["PATH"] = str(bindir) + os.pathsep + env.get("PATH", "")
+    env["PG_AS"] = _posix(bindir / "asuser")
-    env.pop("KEEP", None)
+    for k in ("KEEP", "PS_BACKUP_CONFIG"):
-    env.pop("PS_BACKUP_CONFIG", None)
+        env.pop(k, None)
    return {"env": env, "log": log, "tmp": tmp_path}
-def run_backup(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
+def _run(script: Path, fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
    env = dict(fake["env"])  # type: ignore[call-overload]
    env.update(extra_env)
    assert BASH is not None
-    return subprocess.run([BASH, _posix(BACKUP), *args], capture_output=True, text=True, env=env, timeout=120)
+    return subprocess.run([BASH, _posix(script), *args], capture_output=True, text=True, env=env, timeout=120)
-def stack_args(name: str, folder: Path, container: str = "project:packetstation-test") -> list[str]:
+def run_backup(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
-    return ["--stack", name, container, "packetstation", _posix(folder)]
+    return _run(BACKUP, fake, *args, **extra_env)
 def run_restore(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
    return _run(RESTORE, fake, *args, **extra_env)
 def stack_args(name: str, folder: Path, database: str = "packetstation") -> list[str]:
    return ["--stack", name, database, _posix(folder)]
 def dumps(folder: Path) -> list[str]:
@@ -138,6 +171,10 @@ def no_partials(folder: Path) -> bool:
 MARKER = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z (ps-\d{8}T\d{6}Z\.dump)(.*)$")
 # ---------------------------------------------------------------- ps-backup
@needs_bash
 def test_success_writes_dump_globals_and_marker(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
    res = run_backup(fake, *stack_args("test", folder))
@@ -157,12 +194,14 @@ def test_success_writes_dump_globals_and_marker(fake: dict[str, object]) -> None
        assert stat.S_IMODE(globals_file.stat().st_mode) == 0o600
        assert stat.S_IMODE((folder / "last-success").stat().st_mode) == 0o644
    calls = fake["log"].read_text()  # type: ignore[attr-defined]
-    assert "label=com.docker.compose.project=packetstation-test" in calls
+    # pg_dump and pg_dumpall go through the user switch; pg_restore --list needs no database and no switch.
-    assert "exec pgfake pg_dump -U postgres -d packetstation -Fc" in calls
+    assert re.search(r"^asuser \S*/pg_dump -d packetstation -Fc$", calls, re.M), calls
-    assert "pg_dumpall -U postgres --globals-only --no-role-passwords" in calls
+    assert re.search(r"^asuser \S*/pg_dumpall --globals-only --no-role-passwords$", calls, re.M), calls
-    assert "exec -i pgfake pg_restore --list" in calls
+    assert re.search(r"^pg_restore --list$", calls, re.M), calls
    assert "docker" not in calls
@needs_bash
 def test_rotation_keeps_seven(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
    folder.mkdir()
@@ -183,6 +222,7 @@ def test_rotation_keeps_seven(fake: dict[str, object]) -> None:
    assert sorted(p.name for p in folder.glob("*.globals.sql")) == [n.replace(".dump", ".globals.sql") for n in names]
@needs_bash
 def test_keep_is_configurable(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
    folder.mkdir()
@@ -191,8 +231,10 @@ def test_keep_is_configurable(fake: dict[str, object]) -> None:
    res = run_backup(fake, "--keep", "2", *stack_args("test", folder))
    assert res.returncode == 0, res.stderr
    assert len(dumps(folder)) == 2
    assert run_backup(fake, "--keep", "0", *stack_args("test", folder)).returncode == 2
@needs_bash
 def test_failed_restore_list_writes_last_failure(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
    folder.mkdir()
@@ -212,39 +254,39 @@ def test_failed_restore_list_writes_last_failure(fake: dict[str, object]) -> Non
    assert "FAILED" in res.stderr
@needs_bash
 def test_failed_pg_dump_and_empty_dump(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
-    res = run_backup(fake, *stack_args("test", folder, container="pgdirect"), FAKE_DUMP_FAIL_CONTAINER="pgdirect")
+    res = run_backup(fake, *stack_args("test", folder, database="nosuchdb"), FAKE_DUMP_FAIL_DB="nosuchdb")
    assert res.returncode != 0
    assert "pg_dump failed" in (folder / "last-failure").read_text()
    assert dumps(folder) == []
    assert no_partials(folder)
-    res = run_backup(fake, *stack_args("test", folder, container="pgdirect"), FAKE_DUMP_EMPTY="1")
+    res = run_backup(fake, *stack_args("test", folder), FAKE_DUMP_EMPTY="1")
    assert res.returncode != 0
    assert "pg_dump wrote nothing" in (folder / "last-failure").read_text()
    assert not (folder / "last-success").exists()
-def test_no_container_for_project(fake: dict[str, object]) -> None:
+@needs_bash
 def test_bad_database_name_is_refused(fake: dict[str, object]) -> None:
    folder = fake["tmp"] / "backups"  # type: ignore[operator]
-    res = run_backup(fake, *stack_args("test", folder), FAKE_PS_IDS="")
+    res = run_backup(fake, *stack_args("test", folder, database="x;rm -rf"))
    assert res.returncode != 0
-    assert "no running postgres container" in (folder / "last-failure").read_text()
+    assert "not a plain name" in (folder / "last-failure").read_text()
-
+    assert "pg_dump" not in fake["log"].read_text()  # type: ignore[attr-defined]
    res = run_backup(fake, *stack_args("test", folder), FAKE_PS_IDS="aaa bbb")
    assert res.returncode != 0
    assert "more than one postgres container" in (folder / "last-failure").read_text()
@needs_bash
 def test_one_stack_fails_the_other_still_runs(fake: dict[str, object]) -> None:
    good = fake["tmp"] / "good"  # type: ignore[operator]
    bad = fake["tmp"] / "bad"  # type: ignore[operator]
    res = run_backup(
        fake,
-        *stack_args("bad", bad, container="pgbad"),
+        *stack_args("bad", bad, database="baddb"),
-        *stack_args("good", good, container="pggood"),
+        *stack_args("good", good),
-        FAKE_DUMP_FAIL_CONTAINER="pgbad",
+        FAKE_DUMP_FAIL_DB="baddb",
    )
    assert res.returncode == 1
    assert (bad / "last-failure").exists() and not (bad / "last-success").exists()
@@ -252,15 +294,13 @@ def test_one_stack_fails_the_other_still_runs(fake: dict[str, object]) -> None:
    assert len(dumps(good)) == 1
@needs_bash
 def test_config_file_and_stack_selection(fake: dict[str, object]) -> None:
    tmp: Path = fake["tmp"]  # type: ignore[assignment]
    a, b = tmp / "a", tmp / "b"
    conf = tmp / "backup.conf"
    conf.write_text(
-        "# name container database folder\n"
+        f"# name database folder\nalpha  packetstation  {_posix(a)}\n\nbeta   packetstation  {_posix(b)}\n",
        f"alpha  pga  packetstation  {_posix(a)}\n"
        "\n"
        f"beta   pgb  packetstation  {_posix(b)}\n",
        encoding="utf-8",
        newline="\n",
    )
@@ -275,13 +315,382 @@ def test_config_file_and_stack_selection(fake: dict[str, object]) -> None:
    res = run_backup(fake, "--config", _posix(tmp / "missing.conf"))
    assert res.returncode == 2
    # The old docker-1 format (with a container column) is refused, not misread.
    conf.write_text(f"test project:packetstation-test packetstation {_posix(a)}\n", encoding="utf-8", newline="\n")
    res = run_backup(fake, "--config", _posix(conf))
    assert res.returncode == 2
    assert "name database folder" in res.stderr
-def test_scripts_are_strict_and_parse() -> None:
+
 # ---------------------------------------------------------------- ps-restore
@needs_bash
 def test_restore_into_a_fresh_database(fake: dict[str, object]) -> None:
    tmp: Path = fake["tmp"]  # type: ignore[assignment]
    dump = tmp / "ps-20261030T080000Z.dump"
    dump.write_text("PGDMP fake")
    res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored")
    assert res.returncode == 0, res.stderr
    calls = fake["log"].read_text()  # type: ignore[attr-defined]
    assert "CREATE DATABASE %I OWNER %I ENCODING %L LOCALE %L TEMPLATE template0" in calls
    assert "-v name=ps_restored -v owner=ps_migrate -v loc=en_US.UTF-8" in calls
    assert "GRANT CONNECT, TEMPORARY" in calls
    assert re.search(r"^asuser \S*/pg_restore -d ps_restored --exit-on-error --single-transaction$", calls, re.M)
    assert "packets\t3" in res.stdout
    assert "DROP DATABASE" not in calls
@needs_bash
 def test_restore_guards(fake: dict[str, object]) -> None:
    tmp: Path = fake["tmp"]  # type: ignore[assignment]
    dump = tmp / "ps-1.dump"
    dump.write_text("PGDMP fake")
    res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored", FAKE_DB_EXISTS="1")
    assert res.returncode == 1
    assert "exists. Give --replace" in res.stderr
    res = run_restore(fake, "--dump", _posix(dump), "--db", "packetstation", "--replace", FAKE_DB_EXISTS="1")
    assert res.returncode == 1
    assert "refused for packetstation" in res.stderr
    res = run_restore(fake, "--dump", _posix(dump), "--db", "Bad-Name")
    assert res.returncode == 1
    hashed = tmp / "ps-1.globals.sql"
    hashed.write_text("CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN PASSWORD 'SCRAM-SHA-256$4096:abc';\n")
    res = run_restore(fake, "--globals", _posix(hashed))
    assert res.returncode == 1
    assert "password hashes" in res.stderr
    res = run_restore(fake)
    assert res.returncode == 1
    assert "nothing to do" in res.stderr
@needs_bash
 def test_restore_replace_drops_first(fake: dict[str, object]) -> None:
    dump = fake["tmp"] / "ps-1.dump"  # type: ignore[operator]
    dump.write_text("PGDMP fake")
    res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored", "--replace", FAKE_DB_EXISTS="1")
    assert res.returncode == 0, res.stderr
    calls = fake["log"].read_text()  # type: ignore[attr-defined]
    assert calls.index("DROP DATABASE") < calls.index("CREATE DATABASE")
@needs_bash
 def test_restore_globals_skips_existing_roles(fake: dict[str, object]) -> None:
    g = fake["tmp"] / "ps-1.globals.sql"  # type: ignore[operator]
    g.write_text("CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN;\n")
    res = run_restore(fake, "--globals", _posix(g), FAKE_GLOBALS_ERR='ERROR:  role "ps_app" already exists')
    assert res.returncode == 0, res.stderr
    assert "1 statements skipped" in res.stderr
    res = run_restore(fake, "--globals", _posix(g), FAKE_GLOBALS_ERR="ERROR:  syntax error")
    assert res.returncode == 1
@needs_bash
 def test_restore_assets_never_overwrite_and_verify(fake: dict[str, object]) -> None:
    tmp: Path = fake["tmp"]  # type: ignore[assignment]
    src = tmp / "src"
    (src / "ab").mkdir(parents=True)
    (src / "ab" / "one").write_bytes(b"one")
    (src / "ab" / "two").write_bytes(b"two")
    dst = tmp / "assets"
    (dst / "ab").mkdir(parents=True)
    (dst / "ab" / "two").write_bytes(b"kept")  # an existing file is never replaced
    # 0:0 so a run as root (CI containers) has an owner that exists; as a normal user there is no chown.
    res = run_restore(fake, "--assets", _posix(src), "--assets-dir", _posix(dst), "--assets-owner", "0:0")
    assert res.returncode == 0, res.stderr
    assert (dst / "ab" / "one").read_bytes() == b"one"
    assert (dst / "ab" / "two").read_bytes() == b"kept"
    def rows(files: dict[str, bytes]) -> str:
        return "".join(f"{hashlib.sha256(v).hexdigest()}  {k}\n" for k, v in sorted(files.items()))
    rows_file = tmp / "rows"
    rows_file.write_text(rows({"ab/one": b"one", "ab/two": b"kept"}), newline="\n")
    args = ("--verify-assets", "--db", "ps_restored", "--assets-dir", _posix(dst))
    res = run_restore(fake, *args, FAKE_ASSET_ROWS=_posix(rows_file))
    assert res.returncode == 0, res.stderr
    assert "2 rows, 2 match, 0 differ, 0 missing, 0 files with no row" in res.stderr
    (dst / "ab" / "stray").write_bytes(b"x")
    rows_file.write_text(rows({"ab/one": b"one", "ab/two": b"two", "ab/gone": b"g"}), newline="\n")
    res = run_restore(fake, *args, FAKE_ASSET_ROWS=_posix(rows_file))
    assert res.returncode == 1
    assert "3 rows, 1 match, 1 differ, 1 missing, 1 files with no row" in res.stderr
    assert "ORPHAN: ab/stray" in res.stderr
 # ---------------------------------------------------------------- deploy/vm: launcher and ps-run
 def _release(tmp: Path) -> Path:
    rel = tmp / "opt" / "sha-abc1234"
    (rel / "deploy" / "vm").mkdir(parents=True)
    (rel / "venv" / "bin").mkdir(parents=True)
    (rel / "static").mkdir()
    shutil.copy(VM / "ps-launch.sh", rel / "deploy" / "vm" / "ps-launch.sh")
    (rel / "deploy" / "vm" / "ps-launch.sh").chmod(0o755)
    _write_exec(rel / "venv" / "bin" / "packetstation", FAKE_PACKETSTATION)
    (rel / "BUILD").write_text("sha-abc1234\n", newline="\n")
    return rel
 def _launch(rel: Path, env: dict[str, str], *args: str) -> dict[str, str]:
    assert BASH is not None
    base = {k: v for k, v in os.environ.items() if not k.endswith("_FILE") and k != "CREDENTIALS_DIRECTORY"}
    base.pop("MFILES_PASSWORD", None)
    base.update(env)
    res = subprocess.run(
        [BASH, _posix(rel / "deploy" / "vm" / "ps-launch.sh"), *args],
        capture_output=True,
        text=True,
        env=base,
        timeout=60,
    )
    assert res.returncode == 0, res.stderr
    return dict(line.split("=", 1) for line in res.stdout.splitlines())
@needs_bash
 def test_launcher_maps_credentials_and_sets_release_paths(tmp_path: Path) -> None:
    rel = _release(tmp_path)
    creds = tmp_path / "creds"
    creds.mkdir()
    (creds / "postgres_app_password").write_text("pw-app")
    (creds / "mfileskit_password").write_text("mf-secret\r\n")
    (creds / "seal_key").write_text("k" * 32)
    out = _launch(
        rel,
        {
            "CREDENTIALS_DIRECTORY": _posix(creds),
            "VISUAL_PASSWORD_FILE": "/etc/packetstation/secrets/visualkit_password",
            "STATIC_DIR": "/elsewhere",
            "PS_BUILD": "wrong",
        },
        "worker",
    )
    c = _posix(creds)
    assert out["args"] == "worker"
    assert out["PG_APP_PASSWORD_FILE"] == f"{c}/postgres_app_password"
    assert out["SEAL_KEY_FILE"] == f"{c}/seal_key"
    assert out["MFILES_PASSWORD_FILE"] == f"{c}/mfileskit_password"
    # Not a credential of this unit: the env file's value stays and nothing is invented.
    assert out["VISUAL_PASSWORD_FILE"] == "/etc/packetstation/secrets/visualkit_password"
    assert out["PG_MIGRATE_PASSWORD_FILE"] == "<unset>"
    # Exported from the file, trailing CR LF dropped (the entrypoint drops a CR; the launcher reads with $(<)).
    assert out["MFILES_PASSWORD"] == "mf-secret"
    # The release wins over the env file for these.
    assert out["STATIC_DIR"] == f"{_posix(rel)}/static"
    assert out["PS_BUILD"] == "sha-abc1234"
    assert out["WORKER_BEAT_FILE"] == "/run/packetstation/worker.beat"
    assert out["TZ"] == "UTC"
@needs_bash
 def test_launcher_without_credentials_defaults_to_app(tmp_path: Path) -> None:
    rel = _release(tmp_path)
    out = _launch(rel, {"MFILES_PASSWORD_FILE": "/nonexistent/mfileskit_password"})
    assert out["args"] == "app"
    assert out["MFILES_PASSWORD"] == "<unset>"
    assert out["PG_APP_PASSWORD_FILE"] == "<unset>"
@needs_bash
 def test_ps_run_passes_env_file_credentials_and_sandbox(tmp_path: Path) -> None:
    rel = _release(tmp_path)
    secrets = tmp_path / "secrets"
    secrets.mkdir()
    (secrets / "postgres_app_password").write_text("x")
    (secrets / "ldap_bind_password").write_text("y")
    env_file = tmp_path / "packetstation.env"
    env_file.write_text("STACK=test\n", newline="\n")
    fake_run = tmp_path / "systemd-run"
    _write_exec(fake_run, FAKE_SYSTEMD_RUN)
    assert BASH is not None
    env = dict(os.environ)
    env.update(
        PS_RUN_ALLOW_NONROOT="1",
        SYSTEMD_RUN=_posix(fake_run),
        PS_ENV_FILE_PATH=_posix(env_file),
        PS_SECRETS_DIR=_posix(secrets),
        PS_CURRENT=_posix(rel),
    )
    res = subprocess.run(
        [BASH, _posix(VM / "ps-run.sh"), "doctor", "--skip", "visual"],
        capture_output=True,
        text=True,
        env=env,
        timeout=60,
    )
    assert res.returncode == 0, res.stderr
    argv = res.stdout.splitlines()
    s = _posix(secrets)
    assert argv[:5] == ["--wait", "--pipe", "--collect", "--quiet", "--service-type=exec"]
    assert "User=packetstation" in argv and "Group=packetstation" in argv
    assert f"EnvironmentFile={_posix(env_file)}" in argv
    assert "ProtectSystem=strict" in argv and "NoNewPrivileges=yes" in argv
    assert f"InaccessiblePaths=-{s}" in argv
    creds = [a for a in argv if a.startswith("LoadCredential=")]
    assert creds == [
        f"LoadCredential=postgres_app_password:{s}/postgres_app_password",
        f"LoadCredential=ldap_bind_password:{s}/ldap_bind_password",
    ]
    assert argv[-4:] == [f"{_posix(rel)}/deploy/vm/ps-launch.sh", "doctor", "--skip", "visual"]
    env.pop("PS_RUN_ALLOW_NONROOT")
    if sys.platform != "win32" and os.geteuid() != 0:
        res = subprocess.run([BASH, _posix(VM / "ps-run.sh"), "doctor"], capture_output=True, text=True, env=env)
        assert res.returncode == 2
        assert "run as root" in res.stderr
 # ---------------------------------------------------------------- static checks
 SCRIPTS = [BACKUP, RESTORE, *sorted(VM.glob("*.sh"))]
@needs_bash
@pytest.mark.parametrize("script", SCRIPTS, ids=lambda p: p.name)
 def test_scripts_are_strict_and_parse(script: Path) -> None:
    assert BASH is not None
    for script in (BACKUP, RESTORE):
    text = script.read_text(encoding="utf-8")
    assert "\r" not in text
    assert text.startswith("#!/usr/bin/env bash\n")
    assert "set -euo pipefail" in text
    if script in (BACKUP, RESTORE):
        assert "umask 077" in text
    if script.name != "make-release.sh":
        assert "docker exec" not in text and "docker run" not in text
    res = subprocess.run([BASH, "-n", _posix(script)], capture_output=True, text=True)
    assert res.returncode == 0, res.stderr
 Unit = dict[str, dict[str, list[str]]]
 def _unit(name: str) -> Unit:
    """A unit file as {section: {key: [values]}}. systemd repeats keys (LoadCredential=, ReadWritePaths=)."""
    out: Unit = {}
    section = ""
    for raw in (UNITS / name).read_text(encoding="utf-8").splitlines():
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        if line.startswith("[") and line.endswith("]"):
            section = line[1:-1]
            out.setdefault(section, {})
            continue
        key, _, value = line.partition("=")
        out[section].setdefault(key.strip(), []).append(value.strip())
    return out
 def _values(u: Unit, section: str, key: str) -> list[str]:
    return u.get(section, {}).get(key, [])
 LAUNCH = "/opt/packetstation/current/deploy/vm/ps-launch.sh"
@pytest.mark.parametrize(
    ("unit", "command", "creds"),
    [
        ("packetstation-migrate.service", "migrate", {"postgres_migrate_password", "postgres_app_password"}),
        (
            "packetstation-app.service",
            "app",
            {"postgres_app_password", "ldap_bind_password", "session_key", "seal_key"},
        ),
        (
            "packetstation-worker.service",
            "worker",
            {"postgres_app_password", "visualkit_password", "mfileskit_password", "ldap_bind_password", "seal_key"},
        ),
    ],
 )
 def test_service_units(unit: str, command: str, creds: set[str]) -> None:
    text = (UNITS / unit).read_text(encoding="utf-8")
    assert "\r" not in text
    cp = _unit(unit)
    svc = "Service"
    assert _values(cp, svc, "ExecStart") == [f"{LAUNCH} {command}"]
    assert _values(cp, svc, "User") == ["packetstation"]
    assert _values(cp, svc, "EnvironmentFile") == ["/etc/packetstation/packetstation.env"]
    for key, want in (
        ("NoNewPrivileges", "yes"),
        ("ProtectSystem", "strict"),
        ("ProtectHome", "yes"),
        ("PrivateTmp", "yes"),
        ("InaccessiblePaths", "/etc/packetstation/secrets"),
        ("CapabilityBoundingSet", ""),
    ):
        assert _values(cp, svc, key) == [want], key
    loaded = {v.split(":", 1)[0] for v in _values(cp, svc, "LoadCredential")}
    assert loaded == creds
    for v in _values(cp, svc, "LoadCredential"):
        name, path = v.split(":", 1)
        assert path == f"/etc/packetstation/secrets/{name}"
    rw = _values(cp, svc, "ReadWritePaths")
    if command == "migrate":
        assert rw == []
    else:
        # The app runs the archive check and the worker copies: both need the archive mount.
        assert rw == ["/var/lib/packetstation/assets", "-/mnt/packetstation-archive"]
        assert "packetstation-migrate.service" in _values(cp, "Unit", "Requires")[0]
 def test_app_unit_needs_the_firewall_table() -> None:
    cp = _unit("packetstation-app.service")
    assert _values(cp, "Service", "ExecStartPre") == ["+/usr/sbin/nft list table inet packetstation"]
    fw = _unit("packetstation-firewall.service")
    assert _values(fw, "Service", "ExecStart") == ["/usr/sbin/nft -f /etc/packetstation/packetstation.nft"]
 def test_launcher_knows_every_credential_the_units_load() -> None:
    launch = (VM / "ps-launch.sh").read_text(encoding="utf-8")
    run = (VM / "ps-run.sh").read_text(encoding="utf-8")
    names: set[str] = set()
    for unit in UNITS.glob("*.service"):
        names |= {v.split(":", 1)[0] for v in _values(_unit(unit.name), "Service", "LoadCredential")}
    for n in names:
        assert re.search(rf"(^|')({n}) [A-Z_]+_FILE'?$", launch, re.M), n
        assert n in run, n
@pytest.mark.parametrize("stack", ["prod", "test"])
 def test_env_examples(stack: str) -> None:
    text = (VM / f"{stack}.env.example").read_text(encoding="utf-8")
    assert "\r" not in text
    values = dict(line.split("=", 1) for line in text.splitlines() if line and not line.startswith("#") and "=" in line)
    assert values["STACK"] == stack
    assert values["PG_HOST"] == "/var/run/postgresql"
    assert values["TRUSTED_PROXY_HOST"] == "192.168.101.12"
    assert values["ASSETS_DIR"] == "/var/lib/packetstation/assets"
    assert values["BACKUP_DIR"] == "/var/backups/packetstation"
    assert "#ARCHIVE_ROOT=/mnt/packetstation-archive" in text
    for k, v in values.items():
        if k.endswith("_FILE"):
            assert v.startswith("/etc/packetstation/secrets/"), k
        assert "PASSWORD=" not in f"{k}=", k
    for k in ("STATIC_DIR", "PS_BUILD", "WORKER_BEAT_FILE", "MFILES_CA_BUNDLE"):
        assert k not in values, k
    if stack == "prod":
        assert (values["VISUAL_SERVER"], values["VISUAL_DATABASE"]) == ("vsql.pesco.com", "PESCOINC")
        assert values["NOTIFY_DRY_RUN"] == "false"
    else:
        assert (values["VISUAL_SERVER"], values["VISUAL_DATABASE"]) == ("'vsql.pesco.com\\DEV'", "PESCOINCDEV")
        assert values["NOTIFY_DRY_RUN"] == "true"
 def test_backup_units() -> None:
    svc = (REPO / "scripts" / "backup" / "packetstation-backup.service").read_text(encoding="utf-8")
    timer = (REPO / "scripts" / "backup" / "packetstation-backup.timer").read_text(encoding="utf-8")
    assert "ExecStart=/usr/local/sbin/ps-backup --config /etc/packetstation/backup.conf" in svc
    assert "docker" not in svc.lower()
    assert "OnCalendar=*-*-* 02:00:00 America/Denver" in timer
    assert "Persistent=true" in timer