Native VM install kit: systemd units, launcher, install and upgrade scripts; backups, restore and doctor without docker exec; DEPLOY.md VM, NPM and NFS sections

Co-Authored-By: Claude Opus 5.5 <[email protected]>

deploy/vm/install.sh

@@ -0,0 +1,510 @@
+#!/usr/bin/env bash
+# Packet Station native VM install. Run by Spencer as root on a new test or production VM, from an
+# unpacked release (make-release.sh). Idempotent: run it again after a change and it only fixes
+# what differs. It never overwrites the env file or a secret, never starts the app or the worker,
+# and never touches VISUAL, M-Files or AD.
+#
+# Usage:
+#   install.sh --stack test|prod [--ca-bundle FILE] [--generate-secrets] [--skip-packages]
+#   install.sh --files-only
+#
+#   --stack S            test or prod. Picks the env file example. Read from the env file when it exists.
+#   --ca-bundle FILE     the PESCO CA chain (PEM, scripts/Export-PescoCaBundle.ps1). Installed as
+#                        /etc/packetstation/ca/pesco-ca.pem and into the system trust store.
+#   --generate-secrets   create the four internal secrets that are missing (both PostgreSQL role
+#                        passwords, session_key, seal_key) from /dev/urandom. Never replaces a file.
+#   --skip-packages      leave apt alone (repositories and packages already in place).
+#   --files-only         only the systemd units, the ps-* commands, backup.conf and the firewall
+#                        table. ps-upgrade runs this from the new release.
+#
+# What it does, in order:
+#   1. apt: PostgreSQL 16 (PGDG), ODBC Driver 18 (Microsoft), Python 3.12, nftables, nfs-common.
+#   2. User and group packetstation, uid and gid 10001 (the image's uid, and the NFS share's).
+#   3. Folders: /etc/packetstation, /var/lib/packetstation/assets, /var/backups/packetstation,
+#      /opt/packetstation, the archive mount point /mnt/packetstation-archive.
+#   4. /etc/packetstation/packetstation.env from the stack's example, if missing.
+#   5. The PESCO CA, when --ca-bundle is given.
+#   6. Secret files: owner root:packetstation, mode 0440, one line, no carriage return.
+#   7. PostgreSQL: socket only (conf.d/packetstation.conf), scram for ps_app and ps_migrate on the
+#      socket (pg_hba.conf), the roles and the database as deploy/postgres/initdb makes them.
+#   8. Units, commands, backup timer, firewall (the files-only part).
+set -euo pipefail
+umask 022
+
+PS_USER=packetstation
+PS_ID=10001
+PG_VERSION=16
+PG_CLUSTER=main
+ETC=/etc/packetstation
+SECRETS="$ETC/secrets"
+ENV_FILE="$ETC/packetstation.env"
+CA_FILE="$ETC/ca/pesco-ca.pem"
+NFT_FILE="$ETC/packetstation.nft"
+BACKUP_CONF="$ETC/backup.conf"
+ASSETS=/var/lib/packetstation/assets
+BACKUPS=/var/backups/packetstation
+ARCHIVE_MOUNT=/mnt/packetstation-archive
+OPT=/opt/packetstation
+SBIN=/usr/local/sbin
+UNIT_DIR=/etc/systemd/system
+PG_ETC="/etc/postgresql/$PG_VERSION/$PG_CLUSTER"
+PG_UNIT="postgresql@$PG_VERSION-$PG_CLUSTER.service"
+PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin"
+DB_LOCALE=en_US.UTF-8
+
+# Signing keys, checked by fingerprint before apt trusts them.
+PGDG_KEY_URL=https://www.postgresql.org/media/keys/ACCC4CF8.asc
+PGDG_KEY_FPR=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+MS_KEY_URL=https://packages.microsoft.com/keys/microsoft.asc
+MS_KEY_FPR=BC528686B50D79E339D3721CEB3E94ADBE1229CF
+
+ALL_SECRETS=(postgres_migrate_password postgres_app_password visualkit_password mfileskit_password
+             ldap_bind_password session_key seal_key)
+GENERATED_SECRETS=(postgres_migrate_password postgres_app_password session_key seal_key)
+
+here="$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd -P)"
+release="$(cd -- "$here/../.." && pwd -P)"
+
+log() { printf 'install: %s\n' "$*" >&2; }
+die() { printf 'install: ERROR: %s\n' "$*" >&2; exit 1; }
+usage() { sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+
+stack=""
+ca_bundle=""
+generate=0
+skip_packages=0
+files_only=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --stack) [[ $# -ge 2 ]] || usage; stack="$2"; shift 2 ;;
+    --ca-bundle) [[ $# -ge 2 ]] || usage; ca_bundle="$2"; shift 2 ;;
+    --generate-secrets) generate=1; shift ;;
+    --skip-packages) skip_packages=1; shift ;;
+    --files-only) files_only=1; shift ;;
+    -h|--help) usage ;;
+    *) log "unknown argument $1"; usage ;;
+  esac
+done
+
+[[ "$(id -u)" == "0" ]] || die "run as root"
+[[ -z "$stack" || "$stack" == "test" || "$stack" == "prod" ]] || die "--stack must be test or prod"
+
+# ------------------------------------------------------------------ helpers
+
+# A KEY=value from the env file, with surrounding quotes removed. Empty when unset.
+env_value() {
+  local key="$1" line value
+  [[ -r "$ENV_FILE" ]] || return 0
+  line="$(grep -E "^[[:space:]]*$key=" "$ENV_FILE" | tail -n 1 || true)"
+  value="${line#*=}"
+  value="${value%$'\r'}"
+  value="${value#\"}"; value="${value%\"}"
+  value="${value#\'}"; value="${value%\'}"
+  printf '%s' "$value"
+}
+
+# Install a text file only when it differs. Strips carriage returns (a Windows checkout).
+# Prints "changed" when it wrote the file.
+put_file() {
+  local src="$1" dst="$2" owner="$3" mode="$4" tmp
+  tmp="$(mktemp)"
+  sed 's/\r$//' -- "$src" > "$tmp"
+  if [[ -f "$dst" ]] && cmp -s -- "$tmp" "$dst"; then
+    chown "$owner" -- "$dst"; chmod "$mode" -- "$dst"
+    rm -f -- "$tmp"
+    return 0
+  fi
+  install -o "${owner%%:*}" -g "${owner##*:}" -m "$mode" -- "$tmp" "$dst"
+  rm -f -- "$tmp"
+  echo changed
+}
+
+as_postgres() {
+  setpriv --reuid=postgres --regid=postgres --init-groups -- "$@"
+}
+
+is_ipv4() { [[ "$1" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; }
+is_ipv6() { [[ "$1" =~ ^[0-9A-Fa-f:]+$ && "$1" == *:* ]]; }
+
+# ------------------------------------------------------------------ 1. packages
+
+os_id=""
+os_version=""
+os_codename=""
+read_os() {
+  # shellcheck disable=SC1091
+  os_id="$(. /etc/os-release && printf '%s' "${ID:-}")"
+  # shellcheck disable=SC1091
+  os_version="$(. /etc/os-release && printf '%s' "${VERSION_ID:-}")"
+  # shellcheck disable=SC1091
+  os_codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")"
+}
+
+fetch_key() {
+  local url="$1" fpr="$2" dst="$3" tmp found
+  tmp="$(mktemp -d)"
+  curl -fsSL --proto '=https' "$url" -o "$tmp/key.asc"
+  found="$(gpg --batch --show-keys --with-colons "$tmp/key.asc" 2>/dev/null | awk -F: '$1 == "fpr" && !seen { print $10; seen = 1 }')"
+  if [[ "$found" != "$fpr" ]]; then
+    rm -rf -- "$tmp"
+    die "the key from $url has fingerprint '${found:-none}', expected $fpr. Check it before going on."
+  fi
+  gpg --batch --yes --dearmor -o "$tmp/key.gpg" "$tmp/key.asc"
+  install -o root -g root -m 0644 "$tmp/key.gpg" "$dst"
+  rm -rf -- "$tmp"
+}
+
+step_packages() {
+  local ms_repo python_pkgs=()
+  case "$os_id $os_version" in
+    "ubuntu 24.04")
+      ms_repo="https://packages.microsoft.com/ubuntu/24.04/prod $os_codename main"
+      python_pkgs=(python3.12 python3.12-venv) ;;
+    "debian 12")
+      ms_repo="https://packages.microsoft.com/debian/12/prod $os_codename main"
+      log "Debian 12 ships Python 3.11. Packet Station needs 3.12: give ps-upgrade PYTHON=/path/to/python3.12." ;;
+    *) die "unsupported OS '$os_id $os_version'. Supported: Ubuntu 24.04, Debian 12." ;;
+  esac
+
+  export DEBIAN_FRONTEND=noninteractive
+  log "apt: base packages"
+  apt-get update -q
+  apt-get install -y -q --no-install-recommends \
+    ca-certificates curl gnupg locales tzdata nftables nfs-common util-linux coreutils findutils
+
+  # The database locale. Generated before PostgreSQL so its first cluster knows it too.
+  if ! locale -a 2>/dev/null | grep -qiE '^en_US\.utf-?8$'; then
+    log "generating locale $DB_LOCALE"
+    if [[ -f /etc/locale.gen ]]; then
+      sed -i 's/^# *\(en_US.UTF-8 UTF-8\)/\1/' /etc/locale.gen
+    fi
+    locale-gen en_US.UTF-8 >/dev/null
+  fi
+
+  log "apt: PGDG and Microsoft repositories"
+  fetch_key "$PGDG_KEY_URL" "$PGDG_KEY_FPR" /usr/share/keyrings/pgdg.gpg
+  fetch_key "$MS_KEY_URL" "$MS_KEY_FPR" /usr/share/keyrings/microsoft-prod.gpg
+  printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/pgdg.gpg] https://apt.postgresql.org/pub/repos/apt %s-pgdg main\n' \
+    "$os_codename" > /etc/apt/sources.list.d/pgdg.list
+  printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] %s\n' "$ms_repo" \
+    > /etc/apt/sources.list.d/microsoft-prod.list
+  apt-get update -q
+
+  log "apt: PostgreSQL $PG_VERSION, ODBC Driver 18, Python"
+  ACCEPT_EULA=Y apt-get install -y -q --no-install-recommends \
+    "postgresql-$PG_VERSION" "postgresql-client-$PG_VERSION" msodbcsql18 unixodbc "${python_pkgs[@]}"
+  odbcinst -q -d -n "ODBC Driver 18 for SQL Server" >/dev/null \
+    || die "ODBC Driver 18 for SQL Server is not registered with unixODBC"
+}
+
+# ------------------------------------------------------------------ 2. user
+
+step_user() {
+  local g u
+  g="$(getent group "$PS_USER" | cut -d: -f3 || true)"
+  if [[ -z "$g" ]]; then
+    if getent group "$PS_ID" >/dev/null; then
+      die "gid $PS_ID is taken by group $(getent group "$PS_ID" | cut -d: -f1)"
+    fi
+    groupadd --system --gid "$PS_ID" "$PS_USER"
+    log "created group $PS_USER ($PS_ID)"
+  elif [[ "$g" != "$PS_ID" ]]; then
+    die "group $PS_USER has gid $g, expected $PS_ID (the NFS share and the backups expect $PS_ID)"
+  fi
+  u="$(id -u "$PS_USER" 2>/dev/null || true)"
+  if [[ -z "$u" ]]; then
+    if getent passwd "$PS_ID" >/dev/null; then
+      die "uid $PS_ID is taken by user $(getent passwd "$PS_ID" | cut -d: -f1)"
+    fi
+    useradd --system --uid "$PS_ID" --gid "$PS_ID" --home-dir /var/lib/packetstation --no-create-home \
+      --shell /usr/sbin/nologin --comment "Packet Station" "$PS_USER"
+    log "created user $PS_USER ($PS_ID)"
+  elif [[ "$u" != "$PS_ID" ]]; then
+    die "user $PS_USER has uid $u, expected $PS_ID"
+  fi
+}
+
+# ------------------------------------------------------------------ 3. folders
+
+step_dirs() {
+  install -d -o root -g "$PS_USER" -m 0750 "$ETC" "$SECRETS"
+  install -d -o root -g root -m 0755 "$ETC/ca" "$OPT"
+  install -d -o root -g "$PS_USER" -m 0750 /var/lib/packetstation "$BACKUPS"
+  install -d -o "$PS_USER" -g "$PS_USER" -m 0750 "$ASSETS"
+  # The mount point stays root's and read-only to packetstation, so while the share is not mounted
+  # nothing can be written into the VM's own disk by mistake.
+  if mountpoint -q "$ARCHIVE_MOUNT" 2>/dev/null; then
+    log "$ARCHIVE_MOUNT is mounted; left as it is"
+  else
+    install -d -o root -g root -m 0755 "$ARCHIVE_MOUNT"
+  fi
+}
+
+# ------------------------------------------------------------------ 4. env file
+
+step_env() {
+  local example have
+  if [[ -e "$ENV_FILE" ]]; then
+    have="$(env_value STACK)"
+    if [[ -n "$stack" && "$have" != "$stack" ]]; then
+      die "$ENV_FILE says STACK=$have, but --stack $stack was given"
+    fi
+    chown root:"$PS_USER" "$ENV_FILE"; chmod 0640 "$ENV_FILE"
+    log "kept $ENV_FILE (STACK=$have)"
+    return 0
+  fi
+  [[ -n "$stack" ]] || die "no $ENV_FILE yet: give --stack test or --stack prod"
+  example="$here/$stack.env.example"
+  [[ -r "$example" ]] || die "missing $example"
+  put_file "$example" "$ENV_FILE" "root:$PS_USER" 0640 >/dev/null
+  log "wrote $ENV_FILE from $(basename -- "$example"). Read it through before the first start."
+}
+
+# ------------------------------------------------------------------ 5. CA
+
+step_ca() {
+  if [[ -z "$ca_bundle" ]]; then
+    [[ -f "$CA_FILE" ]] || log "no PESCO CA yet: run again with --ca-bundle FILE (LDAP and SMTP need it)"
+    return 0
+  fi
+  [[ -r "$ca_bundle" ]] || die "cannot read $ca_bundle"
+  grep -q 'BEGIN CERTIFICATE' "$ca_bundle" || die "$ca_bundle holds no PEM certificate"
+  if grep -q 'PRIVATE KEY' "$ca_bundle"; then
+    die "$ca_bundle holds a private key. Export only the certificates."
+  fi
+  put_file "$ca_bundle" "$CA_FILE" root:root 0644 >/dev/null
+  # update-ca-certificates wants one certificate per .crt file.
+  local dir=/usr/local/share/ca-certificates/pesco tmp
+  tmp="$(mktemp -d)"
+  sed 's/\r$//' -- "$CA_FILE" | awk -v d="$tmp" '
+    /-----BEGIN CERTIFICATE-----/ { n++; f = sprintf("%s/pesco-ca-%02d.crt", d, n) }
+    f { print > f }
+    /-----END CERTIFICATE-----/ { close(f); f = "" }'
+  rm -rf -- "$dir"
+  install -d -o root -g root -m 0755 "$dir"
+  install -o root -g root -m 0644 "$tmp"/*.crt "$dir"/
+  rm -rf -- "$tmp"
+  update-ca-certificates >/dev/null
+  log "PESCO CA installed: $CA_FILE and the system trust store ($(find "$dir" -name '*.crt' | wc -l) certificates)"
+}
+
+# ------------------------------------------------------------------ 6. secrets
+
+new_secret() {
+  # 32 random bytes as 64 hex characters. Held in a variable and written by a builtin, never on argv.
+  local name="$1" value tmp
+  value="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')"
+  [[ ${#value} -eq 64 ]] || die "could not read /dev/urandom"
+  tmp="$(mktemp "$SECRETS/.$name.XXXXXX")"
+  printf '%s' "$value" > "$tmp"
+  unset value
+  chown root:"$PS_USER" "$tmp"; chmod 0440 "$tmp"
+  mv -- "$tmp" "$SECRETS/$name"
+  log "generated secret $name"
+}
+
+missing_secrets=()
+step_secrets() {
+  local name f g
+  missing_secrets=()
+  for name in "${ALL_SECRETS[@]}"; do
+    f="$SECRETS/$name"
+    if [[ ! -e "$f" && $generate -eq 1 ]]; then
+      for g in "${GENERATED_SECRETS[@]}"; do
+        if [[ "$g" == "$name" ]]; then new_secret "$name"; fi
+      done
+    fi
+    if [[ ! -e "$f" ]]; then
+      missing_secrets+=("$name")
+      continue
+    fi
+    chown root:"$PS_USER" -- "$f"; chmod 0440 -- "$f"
+    [[ -s "$f" ]] || die "secret $name is empty"
+    if [[ "$(wc -l < "$f")" -gt 1 ]] || grep -q $'\r' -- "$f"; then
+      die "secret $name must be one line with no carriage return"
+    fi
+  done
+  if [[ ${#missing_secrets[@]} -gt 0 ]]; then
+    log "secrets still missing in $SECRETS: ${missing_secrets[*]}"
+  fi
+}
+
+have_secret() { [[ -s "$SECRETS/$1" ]]; }
+
+# ------------------------------------------------------------------ 7. PostgreSQL
+
+step_postgres() {
+  local changed="" hba="$PG_ETC/pg_hba.conf" tmp
+  [[ -d "$PG_ETC" ]] || die "no PostgreSQL cluster $PG_VERSION/$PG_CLUSTER in $PG_ETC"
+  grep -Eq "^[[:space:]]*include_dir[[:space:]]*=[[:space:]]*'conf\.d'" "$PG_ETC/postgresql.conf" \
+    || die "$PG_ETC/postgresql.conf has no include_dir = 'conf.d'"
+  install -d -o postgres -g postgres -m 0755 "$PG_ETC/conf.d"
+  changed="$(put_file "$here/postgres/packetstation.conf" "$PG_ETC/conf.d/packetstation.conf" postgres:postgres 0644)"
+
+  # ps_app and ps_migrate log in on the socket with their passwords (scram). The block goes first,
+  # before Debian's "local all all peer", which would refuse them (the OS user is packetstation).
+  if ! grep -q '^# packetstation begin' "$hba"; then
+    tmp="$(mktemp)"
+    awk '
+      !done && $0 !~ /^[[:space:]]*(#|$)/ {
+        print "# packetstation begin (deploy/vm/install.sh)"
+        print "local   packetstation   ps_app,ps_migrate   scram-sha-256"
+        print "# packetstation end"
+        print ""
+        done = 1
+      }
+      { print }' "$hba" > "$tmp"
+    install -o postgres -g postgres -m 0640 "$tmp" "$hba"
+    rm -f -- "$tmp"
+    log "pg_hba.conf: ps_app and ps_migrate use scram-sha-256 on the socket"
+    [[ -n "$changed" ]] || changed=reload
+  fi
+
+  systemctl enable --quiet "$PG_UNIT" 2>/dev/null || true
+  if [[ "$changed" == "changed" ]]; then
+    log "restarting PostgreSQL (conf.d/packetstation.conf changed)"
+    systemctl restart "$PG_UNIT"
+  elif [[ "$changed" == "reload" ]]; then
+    systemctl start "$PG_UNIT"
+    systemctl reload "$PG_UNIT"
+  else
+    systemctl start "$PG_UNIT"
+  fi
+
+  if ! have_secret postgres_migrate_password || ! have_secret postgres_app_password; then
+    log "skipping roles and database: the PostgreSQL secrets are missing (or use --generate-secrets)"
+    return 0
+  fi
+
+  log "PostgreSQL roles ps_migrate and ps_app, database packetstation"
+  # The passwords reach psql through its environment (\getenv), never through argv or a file.
+  # A failed statement is not logged with its text (log_min_error_statement), so a password
+  # cannot land in the server log.
+  PS_MIGRATE_PW="$(<"$SECRETS/postgres_migrate_password")"
+  PS_APP_PW="$(<"$SECRETS/postgres_app_password")"
+  export PS_MIGRATE_PW PS_APP_PW
+  as_postgres "$PG_BIN/psql" -X -q -v ON_ERROR_STOP=1 -d postgres <<SQL
+SET log_min_error_statement = panic;
+SET log_statement = 'none';
+\\getenv migrate_pw PS_MIGRATE_PW
+\\getenv app_pw PS_APP_PW
+SELECT 'CREATE ROLE ps_migrate' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_migrate') \\gexec
+SELECT 'CREATE ROLE ps_app' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_app') \\gexec
+ALTER ROLE ps_migrate WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'migrate_pw';
+ALTER ROLE ps_app     WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'app_pw';
+\\unset migrate_pw
+\\unset app_pw
+SELECT format('CREATE DATABASE packetstation OWNER ps_migrate ENCODING %L LOCALE %L TEMPLATE template0',
+              'UTF8', '$DB_LOCALE')
+ WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'packetstation') \\gexec
+REVOKE ALL ON DATABASE packetstation FROM PUBLIC;
+GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app;
+\\connect packetstation
+REVOKE ALL ON SCHEMA public FROM PUBLIC;
+ALTER SCHEMA public OWNER TO ps_migrate;
+SQL
+  unset PS_MIGRATE_PW PS_APP_PW
+}
+
+# ------------------------------------------------------------------ 8. files
+
+write_firewall() {
+  local proxy rule tmp
+  proxy="$(env_value TRUSTED_PROXY_HOST)"
+  if is_ipv4 "$proxy"; then
+    rule="ip saddr $proxy"
+  elif is_ipv6 "$proxy"; then
+    rule="ip6 saddr $proxy"
+  else
+    die "TRUSTED_PROXY_HOST in $ENV_FILE must be NPM's IP address (it is '${proxy}')"
+  fi
+  tmp="$(mktemp)"
+  cat > "$tmp" <<NFT
+#!/usr/sbin/nft -f
+# Written by deploy/vm/install.sh from TRUSTED_PROXY_HOST in $ENV_FILE. Do not edit:
+# change the env file and run install.sh --files-only. Loaded by packetstation-firewall.service.
+# Port 8000 (the app) answers only NPM on docker-1 and the VM itself. Nothing else is filtered here.
+table inet packetstation
+delete table inet packetstation
+table inet packetstation {
+  chain input {
+    type filter hook input priority filter; policy accept;
+    tcp dport 8000 iifname "lo" accept
+    tcp dport 8000 $rule accept
+    tcp dport 8000 counter drop
+  }
+}
+NFT
+  if command -v nft >/dev/null 2>&1; then
+    nft -c -f "$tmp" || { rm -f -- "$tmp"; die "nft refused the generated table"; }
+  fi
+  put_file "$tmp" "$NFT_FILE" root:root 0644 >/dev/null
+  rm -f -- "$tmp"
+}
+
+step_files() {
+  local f st
+  [[ -r "$ENV_FILE" ]] || die "no $ENV_FILE: run install.sh --stack test|prod first"
+  st="$(env_value STACK)"
+  [[ "$st" == "test" || "$st" == "prod" ]] || die "STACK in $ENV_FILE must be test or prod"
+
+  for f in "$here"/systemd/*.service; do
+    put_file "$f" "$UNIT_DIR/$(basename -- "$f")" root:root 0644 >/dev/null
+  done
+  put_file "$release/scripts/backup/packetstation-backup.service" "$UNIT_DIR/packetstation-backup.service" root:root 0644 >/dev/null
+  put_file "$release/scripts/backup/packetstation-backup.timer" "$UNIT_DIR/packetstation-backup.timer" root:root 0644 >/dev/null
+
+  put_file "$here/ps-run.sh" "$SBIN/ps-run" root:root 0755 >/dev/null
+  put_file "$here/ps-doctor.sh" "$SBIN/ps-doctor" root:root 0755 >/dev/null
+  put_file "$here/upgrade.sh" "$SBIN/ps-upgrade" root:root 0755 >/dev/null
+  put_file "$release/scripts/backup/ps-backup.sh" "$SBIN/ps-backup" root:root 0755 >/dev/null
+  put_file "$release/scripts/backup/ps-restore.sh" "$SBIN/ps-restore" root:root 0755 >/dev/null
+
+  if [[ ! -e "$BACKUP_CONF" ]]; then
+    printf '# name  database       folder\n%s  packetstation  %s\n' "$st" "$BACKUPS" > "$BACKUP_CONF"
+    chmod 0644 "$BACKUP_CONF"
+    log "wrote $BACKUP_CONF"
+  fi
+
+  write_firewall
+  systemctl daemon-reload
+  systemctl enable --quiet packetstation-firewall.service packetstation-backup.timer
+  if systemctl is-active --quiet packetstation-firewall.service; then
+    systemctl reload packetstation-firewall.service
+  else
+    systemctl start packetstation-firewall.service
+  fi
+  systemctl start packetstation-backup.timer
+  log "units, ps-run, ps-doctor, ps-upgrade, ps-backup, ps-restore and the firewall table are in place"
+}
+
+# ------------------------------------------------------------------ main
+
+if [[ $files_only -eq 1 ]]; then
+  step_files
+  exit 0
+fi
+
+read_os
+if [[ $skip_packages -eq 0 ]]; then
+  step_packages
+fi
+for tool in setpriv systemctl install mountpoint; do
+  command -v "$tool" >/dev/null 2>&1 || die "$tool is not installed"
+done
+step_user
+step_dirs
+step_env
+step_ca
+step_secrets
+step_postgres
+step_files
+
+cat >&2 <<EOF
+install: done for STACK=$(env_value STACK).
+install: next:
+  - read $ENV_FILE through
+  - secrets still missing: ${missing_secrets[*]:-none}
+  - install a release:  ps-upgrade /path/to/packetstation-<build>.tar.gz
+  - the app and the worker are not enabled or started. Starting them is a separate step (docs/DEPLOY.md).
+EOF

deploy/vm/make-release.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Build a release archive for the native VMs. Runs on a build host as a normal user, inside a git
+# checkout of this repository: docker-1 (with --image) or a PC with Node (with --npm). Never on a VM.
+#
+# Usage:
+#   make-release.sh --image gt.pescoinc.biz/spencerm/packetstation:sha-<sha> [--out DIR]
+#   make-release.sh --npm [--ref REF] [--build NAME] [--out DIR]
+#
+#   --image IMAGE   take the built frontend out of a CI-tested image (docker create and docker cp,
+#                   nothing runs). The build name and the commit come from the image's build label,
+#                   so the source and the frontend are the tested commit.
+#   --npm           build the frontend here with npm ci and npm run build, from the same commit.
+#   --ref REF       the commit for --npm (default HEAD). Uncommitted changes are never included.
+#   --build NAME    the build name for --npm (default sha-<full sha>, as CI names images).
+#   --out DIR       where to write the archive (default the current folder).
+#
+# Writes packetstation-<build>.tar.gz and packetstation-<build>.tar.gz.sha256. The archive holds one
+# folder, packetstation-<build>/, with BUILD, pyproject.toml, README.md, server/, deploy/, scripts/,
+# docs/ and static/ (the built frontend). Python packages are not in it: ps-upgrade builds the venv
+# on the VM from the pinned requirements.
+set -euo pipefail
+umask 022
+
+DOCKER="${DOCKER:-docker}"
+LABEL=biz.pescoinc.packetstation.build
+
+log() { printf 'make-release: %s\n' "$*" >&2; }
+die() { printf 'make-release: ERROR: %s\n' "$*" >&2; exit 1; }
+usage() { sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+
+image=""
+npm=0
+ref=""
+build=""
+out="."
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --image) [[ $# -ge 2 ]] || usage; image="$2"; shift 2 ;;
+    --npm) npm=1; shift ;;
+    --ref) [[ $# -ge 2 ]] || usage; ref="$2"; shift 2 ;;
+    --build) [[ $# -ge 2 ]] || usage; build="$2"; shift 2 ;;
+    --out) [[ $# -ge 2 ]] || usage; out="$2"; shift 2 ;;
+    -h|--help) usage ;;
+    *) log "unknown argument $1"; usage ;;
+  esac
+done
+if [[ -n "$image" && $npm -eq 1 ]] || [[ -z "$image" && $npm -eq 0 ]]; then
+  die "give exactly one of --image or --npm"
+fi
+if [[ -n "$image" && ( -n "$ref" || -n "$build" ) ]]; then
+  die "--ref and --build come from the image's label with --image"
+fi
+[[ "$image" != *:latest && ( -z "$image" || "$image" == *:* ) ]] || die "pull an image by its tag, never latest"
+
+repo="$(git rev-parse --show-toplevel 2>/dev/null)" || die "run inside a git checkout of Packet Station"
+cd -- "$repo"
+
+if [[ -n "$image" ]]; then
+  "$DOCKER" image inspect "$image" >/dev/null 2>&1 || "$DOCKER" pull "$image" >/dev/null
+  build="$("$DOCKER" image inspect -f "{{ index .Config.Labels \"$LABEL\" }}" "$image")"
+  [[ "$build" =~ ^sha-[0-9a-f]{7,40}$ ]] || die "image label $LABEL is '$build', expected sha-<commit>"
+  ref="${build#sha-}"
+else
+  ref="${ref:-HEAD}"
+fi
+sha="$(git rev-parse --verify --quiet "$ref^{commit}")" || die "commit $ref is not in this checkout (git fetch first)"
+if [[ -z "$build" ]]; then
+  build="sha-$sha"
+fi
+[[ "$build" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] || die "build name '$build' must be letters, digits, '.', '_' or '-'"
+
+name="packetstation-$build"
+mkdir -p -- "$out"
+out="$(cd -- "$out" && pwd)"
+work="$(mktemp -d)"
+cid=""
+cleanup() {
+  if [[ -n "$cid" ]]; then "$DOCKER" rm -f "$cid" >/dev/null 2>&1 || true; fi
+  rm -rf -- "$work"
+}
+trap cleanup EXIT
+
+log "source of $sha"
+# LF line endings whatever the checkout's autocrlf: these files run on Linux.
+git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix="$name/" "$sha" \
+  pyproject.toml README.md server deploy scripts docs | tar -xf - -C "$work"
+printf '%s\n' "$build" > "$work/$name/BUILD"
+[[ -f "$work/$name/deploy/vm/ps-launch.sh" ]] || die "commit $sha has no deploy/vm (older than phase 10)"
+chmod 0755 "$work/$name"/deploy/vm/*.sh "$work/$name"/scripts/backup/*.sh
+
+if [[ -n "$image" ]]; then
+  log "frontend from $image"
+  cid="$("$DOCKER" create "$image")"
+  "$DOCKER" cp "$cid:/app/static" "$work/$name/static"
+else
+  log "frontend: npm ci and npm run build at $sha"
+  git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix=frontend/ "$sha:frontend" \
+    | tar -xf - -C "$work"
+  ( cd -- "$work/frontend" && npm ci --no-audit --no-fund && npm run build ) >&2
+  cp -R -- "$work/frontend/dist" "$work/$name/static"
+fi
+[[ -f "$work/$name/static/index.html" ]] || die "no static/index.html in the built frontend"
+
+log "writing $out/$name.tar.gz"
+tar -czf "$out/$name.tar.gz" -C "$work" --owner=0 --group=0 --numeric-owner --sort=name "$name"
+( cd -- "$out" && sha256sum -- "$name.tar.gz" > "$name.tar.gz.sha256" )
+log "done: $out/$name.tar.gz ($(wc -c < "$out/$name.tar.gz") bytes) and its .sha256"

deploy/vm/postgres/packetstation.conf

@@ -0,0 +1,13 @@
+# Packet Station on this VM. Installed by deploy/vm/install.sh as
+# /etc/postgresql/16/main/conf.d/packetstation.conf. A change here needs a PostgreSQL restart.
+
+# Local socket only. The app, the worker, migrate and the backups all connect through
+# /var/run/postgresql. Nothing on the network can reach PostgreSQL.
+listen_addresses = ''
+
+# The same as the postgres container on docker-1: the server clock zone is UTC.
+# The app shows times in SITE_TZ.
+timezone = 'UTC'
+log_timezone = 'UTC'
+
+password_encryption = 'scram-sha-256'

deploy/vm/prod.env.example

@@ -0,0 +1,102 @@
+# Packet Station production settings on the production VM (native, no Docker).
+# deploy/vm/install.sh --stack prod copies this to /etc/packetstation/packetstation.env
+# (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
+# It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
+# Starting packetstation-worker with this file polls VISUAL production on its schedule.
+# Spencer's first start of the worker is the approval.
+#
+# systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
+# Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
+# Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
+
+# ---- Stack and site ----
+# prod, test, ci or dev. Guards against crossing stacks.
+STACK=prod
+# The VISUAL site in every call. The docker-1 test stack runs with PESCO.
+SITE_ID=PESCO
+# Site time zone for schedules and display. The processes run in UTC.
+SITE_TZ=America/Denver
+# Public URL, used in links and mail.
+PUBLIC_URL=https://packetstation.pescoinc.biz
+# NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
+# opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
+TRUSTED_PROXY_HOST=192.168.101.12
+# DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
+LOG_LEVEL=INFO
+
+# ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
+# A folder means the Unix socket. PostgreSQL listens on no TCP port.
+PG_HOST=/var/run/postgresql
+PG_PORT=5432
+PG_DATABASE=packetstation
+PG_APP_USER=ps_app
+PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
+PG_MIGRATE_USER=ps_migrate
+PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
+
+# ---- Keys and folders ----
+# The services get the secrets as systemd credentials and ps-launch.sh points these at them.
+# The paths here are the files those credentials are loaded from.
+SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
+SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
+# Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
+ASSETS_DIR=/var/lib/packetstation/assets
+# Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
+BACKUP_DIR=/var/backups/packetstation
+# The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
+# `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
+# Admin on the Admin page, File locations.
+#ARCHIVE_ROOT=/mnt/packetstation-archive
+
+# ---- Uploads and storage ----
+UPLOAD_MAX_BYTES=26214400
+UPLOAD_MAX_BYTES_BOM=5242880
+ASSETS_QUOTA_WARN_PCT=85
+# 0 measures the file system under ASSETS_DIR.
+ASSETS_QUOTA_BYTES=0
+
+# ---- ClamAV (optional, off: Gitea issue 19) ----
+# Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
+#CLAMAV_HOST=127.0.0.1
+CLAMAV_PORT=3310
+CLAMAV_TIMEOUT_SECONDS=30
+
+# ---- LDAP (PESCO Active Directory) ----
+LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
+LDAP_BASE_DN=DC=PESCO,DC=com
+LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
+LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
+LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
+LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
+# The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
+LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
+LDAP_STARTTLS=false
+LDAP_ALLOW_INSECURE=false
+
+# ---- Mail (relay app-03, no auth) ----
+SMTP_HOST=app-03.pesco.com
+SMTP_PORT=25
+SMTP_FROM=[email protected]
+SMTP_STARTTLS=true
+SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
+# Production sends mail.
+NOTIFY_DRY_RUN=false
+
+# ---- VisualKit (VISUAL ERP, read only) ----
+# Production, set explicitly. Never on the test VM.
+VISUAL_SERVER=vsql.pesco.com
+VISUAL_DATABASE=PESCOINC
+VISUAL_USER=visualkit_ro
+VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
+# no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
+# store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
+VISUAL_ENCRYPT=no
+VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
+
+# ---- MFilesKit (M-Files, read only) ----
+MFILES_URL=https://mfiles.pescoinc.biz/REST
+MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
+MFILES_USERNAME=mfileskitRO
+# ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
+MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
+# Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.

deploy/vm/ps-doctor.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# `packetstation doctor` on this VM, run the way the services run (see ps-run.sh).
+# Installed as /usr/local/sbin/ps-doctor. Run as root: sudo ps-doctor [--skip visual,mfiles,ldap,smtp]
+# On production the VISUAL check connects to VISUAL production, read-only, as visualkit_ro.
+set -euo pipefail
+PS_RUN="${PS_RUN:-/usr/local/sbin/ps-run}"
+exec "$PS_RUN" doctor "$@"

deploy/vm/ps-launch.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Packet Station launcher on a VM. The native counterpart of docker/entrypoint.sh.
+# The systemd units and ps-run start every packetstation command through it.
+#
+# Usage: ps-launch.sh app | worker | migrate | doctor [...] | job run <name> | version | healthcheck app|worker | ...
+#
+# It runs from a release folder, /opt/packetstation/<release>/deploy/vm/ps-launch.sh, and uses that
+# release's venv and built frontend. systemd has already loaded /etc/packetstation/packetstation.env.
+#
+# Secrets: each unit hands its own secrets to the service as systemd credentials (LoadCredential=),
+# so the app never sees the VISUAL or M-Files password. When $CREDENTIALS_DIRECTORY holds a known
+# secret, the matching *_FILE setting is pointed at it. MFilesKit reads its password from the
+# environment only, so MFILES_PASSWORD is exported from MFILES_PASSWORD_FILE, as the entrypoint does.
+# No value is ever printed.
+#
+# Set here, over the env file: STATIC_DIR and PS_BUILD (from the release), WORKER_BEAT_FILE, TZ, LANG.
+set -euo pipefail
+
+self="$(readlink -f -- "${BASH_SOURCE[0]}")"
+release="$(cd -- "$(dirname -- "$self")/../.." && pwd -P)"
+venv="${PS_VENV:-$release/venv}"
+
+# secret name (the credential id and the file name in /etc/packetstation/secrets) and its setting
+cred_map='postgres_app_password PG_APP_PASSWORD_FILE
+postgres_migrate_password PG_MIGRATE_PASSWORD_FILE
+visualkit_password VISUAL_PASSWORD_FILE
+mfileskit_password MFILES_PASSWORD_FILE
+ldap_bind_password LDAP_PASSWORD_FILE
+session_key SESSION_KEY_FILE
+seal_key SEAL_KEY_FILE'
+
+if [[ -n "${CREDENTIALS_DIRECTORY:-}" ]]; then
+  while read -r name var; do
+    if [[ -r "$CREDENTIALS_DIRECTORY/$name" ]]; then
+      export "$var=$CREDENTIALS_DIRECTORY/$name"
+    fi
+  done <<< "$cred_map"
+fi
+unset cred_map name var
+
+mfiles_secret="${MFILES_PASSWORD_FILE:-}"
+if [[ -n "$mfiles_secret" && -r "$mfiles_secret" ]]; then
+  MFILES_PASSWORD="$(<"$mfiles_secret")"
+  # Drop a trailing carriage return left by an editor on Windows.
+  MFILES_PASSWORD="${MFILES_PASSWORD%$'\r'}"
+  export MFILES_PASSWORD
+fi
+unset mfiles_secret
+
+build="dev"
+if [[ -r "$release/BUILD" ]]; then
+  build="$(head -n 1 -- "$release/BUILD")"
+  build="${build%$'\r'}"
+fi
+
+export PATH="$venv/bin:/usr/local/bin:/usr/bin:/bin"
+export STATIC_DIR="$release/static"
+export PS_BUILD="$build"
+# The worker unit makes /run/packetstation (RuntimeDirectory=). The heartbeat lives there so
+# `ps-run healthcheck worker` reads the same file the worker writes.
+export WORKER_BEAT_FILE=/run/packetstation/worker.beat
+# The same as the containers: UTC process clock, UTF-8, unbuffered logs to the journal.
+export TZ=UTC
+export LANG=C.UTF-8
+export PYTHONUNBUFFERED=1
+export PYTHONDONTWRITEBYTECODE=1
+unset build self
+
+if [[ $# -eq 0 ]]; then
+  set -- app
+fi
+
+exec "$venv/bin/packetstation" "$@"

deploy/vm/ps-run.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Run one packetstation command on this VM the way the services run it: as the packetstation user,
+# with the env file, the secrets as systemd credentials and the same sandbox, through systemd-run.
+# Output comes back to this terminal and the exit code is the command's.
+#
+# Installed as /usr/local/sbin/ps-run by deploy/vm/install.sh. Run it as root (sudo).
+#
+# Usage:
+#   ps-run doctor [--skip visual,mfiles,ldap,smtp]
+#   ps-run job run visual-feed
+#   ps-run ad-groups --list
+#   ps-run healthcheck worker
+#   ps-run version
+#
+# Every secret file present in /etc/packetstation/secrets is handed over as a credential; the
+# folder itself stays out of reach of the command. No value is read or printed here.
+set -euo pipefail
+
+ENV_FILE="${PS_ENV_FILE_PATH:-/etc/packetstation/packetstation.env}"
+SECRETS_DIR="${PS_SECRETS_DIR:-/etc/packetstation/secrets}"
+CURRENT="${PS_CURRENT:-/opt/packetstation/current}"
+SYSTEMD_RUN="${SYSTEMD_RUN:-systemd-run}"
+SERVICE_USER="${PS_USER:-packetstation}"
+ASSETS_DIR_RW="${PS_ASSETS_DIR:-/var/lib/packetstation/assets}"
+ARCHIVE_DIR_RW="${PS_ARCHIVE_MOUNT:-/mnt/packetstation-archive}"
+
+die() { printf 'ps-run: %s\n' "$*" >&2; exit 2; }
+
+if [[ $# -eq 0 || "$1" == "-h" || "$1" == "--help" ]]; then
+  sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' >&2
+  exit 2
+fi
+if [[ "${PS_RUN_ALLOW_NONROOT:-}" != "1" && "$(id -u)" != "0" ]]; then
+  die "run as root (sudo ps-run ...)"
+fi
+[[ -r "$ENV_FILE" ]] || die "env file $ENV_FILE is missing"
+[[ -x "$CURRENT/deploy/vm/ps-launch.sh" ]] || die "no release at $CURRENT (run ps-upgrade first)"
+
+props=(
+  -p "User=$SERVICE_USER"
+  -p "Group=$SERVICE_USER"
+  -p "EnvironmentFile=$ENV_FILE"
+  -p UMask=0027
+  -p NoNewPrivileges=yes
+  -p ProtectSystem=strict
+  -p ProtectHome=yes
+  -p PrivateTmp=yes
+  -p PrivateDevices=yes
+  -p ProtectKernelTunables=yes
+  -p ProtectKernelModules=yes
+  -p ProtectControlGroups=yes
+  -p RestrictSUIDSGID=yes
+  -p "ReadWritePaths=$ASSETS_DIR_RW"
+  -p "ReadWritePaths=-$ARCHIVE_DIR_RW"
+  -p "InaccessiblePaths=-$SECRETS_DIR"
+)
+for name in postgres_app_password postgres_migrate_password visualkit_password mfileskit_password \
+            ldap_bind_password session_key seal_key; do
+  if [[ -f "$SECRETS_DIR/$name" ]]; then
+    props+=(-p "LoadCredential=$name:$SECRETS_DIR/$name")
+  fi
+done
+
+exec "$SYSTEMD_RUN" --wait --pipe --collect --quiet --service-type=exec \
+  --description="Packet Station: packetstation $1" \
+  "${props[@]}" -- "$CURRENT/deploy/vm/ps-launch.sh" "$@"

deploy/vm/systemd/packetstation-app.service

@@ -0,0 +1,70 @@
+# Packet Station: the web app (uvicorn on port 8000, the API and the built frontend).
+# Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
+# NPM on docker-1 is the only client allowed on port 8000 (packetstation-firewall.service).
+[Unit]
+Description=Packet Station web app
+Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
+Requires=packetstation-migrate.service
+# Wants, not Requires: a restart of nftables.service must not restart the app. ExecStartPre checks the table.
+Wants=packetstation-firewall.service
+After=packetstation-migrate.service packetstation-firewall.service [email protected] network-online.target
+Wants=[email protected] network-online.target
+StartLimitIntervalSec=300
+StartLimitBurst=5
+
+[Service]
+Type=exec
+User=packetstation
+Group=packetstation
+EnvironmentFile=/etc/packetstation/packetstation.env
+# The app's secrets only. It never gets the VISUAL or M-Files password.
+LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
+LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
+LoadCredential=session_key:/etc/packetstation/secrets/session_key
+LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
+# The app never listens without the firewall table in place ("+": this check runs as root).
+ExecStartPre=+/usr/sbin/nft list table inet packetstation
+ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh app
+SyslogIdentifier=packetstation-app
+Restart=on-failure
+RestartSec=5s
+TimeoutStopSec=30s
+WorkingDirectory=/
+UMask=0027
+MemoryMax=1G
+LimitNOFILE=16384
+
+# Sandbox. Read-only file system except the assets and the archive mount.
+# The leading "-" lets the app start while the archive is not mounted (ARCHIVE_ROOT unset).
+ReadWritePaths=/var/lib/packetstation/assets
+ReadWritePaths=-/mnt/packetstation-archive
+InaccessiblePaths=/etc/packetstation/secrets
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+# AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RemoveIPC=yes
+CapabilityBoundingSet=
+AmbientCapabilities=
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+# Not set: MemoryDenyWriteExecute. pypdfium2 goes through ctypes, whose callbacks may need
+# writable and executable memory. Turn it on only after a filing and a print pass on the test VM.
+
+[Install]
+WantedBy=multi-user.target

deploy/vm/systemd/packetstation-firewall.service

@@ -0,0 +1,21 @@
+# Packet Station: only NPM on docker-1 (TRUSTED_PROXY_HOST) and the VM itself may reach port 8000.
+# Loads /etc/packetstation/packetstation.nft, which deploy/vm/install.sh writes from the env file.
+# The table is Packet Station's own (inet packetstation); other rules on the VM are left alone.
+[Unit]
+Description=Packet Station firewall for the app port
+Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
+Before=network-pre.target packetstation-app.service
+Wants=network-pre.target
+# A restart of nftables.service flushes the whole ruleset. Load the table again after it.
+After=nftables.service
+PartOf=nftables.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
+ExecReload=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
+ExecStop=-/usr/sbin/nft delete table inet packetstation
+
+[Install]
+WantedBy=multi-user.target

deploy/vm/systemd/packetstation-migrate.service

@@ -0,0 +1,52 @@
+# Packet Station: SQL migrations, once per boot and once per upgrade, before the app and the worker.
+# Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
+# ps-upgrade restarts this unit after switching /opt/packetstation/current to a new release.
+[Unit]
+Description=Packet Station database migrations
+Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
+Wants=[email protected] network-online.target
+After=[email protected] network-online.target
+Before=packetstation-app.service packetstation-worker.service
+ConditionPathExists=/opt/packetstation/current/deploy/vm/ps-launch.sh
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+User=packetstation
+Group=packetstation
+EnvironmentFile=/etc/packetstation/packetstation.env
+# migrate needs both passwords: it logs in as ps_migrate and grants to ps_app.
+LoadCredential=postgres_migrate_password:/etc/packetstation/secrets/postgres_migrate_password
+LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
+ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh migrate
+SyslogIdentifier=packetstation-migrate
+TimeoutStartSec=15min
+WorkingDirectory=/
+UMask=0027
+
+# Sandbox. The whole file system is read-only; migrate writes nothing outside PostgreSQL.
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+InaccessiblePaths=/etc/packetstation/secrets
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RemoveIPC=yes
+CapabilityBoundingSet=
+AmbientCapabilities=
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+# No [Install]: the app and the worker pull this unit in (Requires=).

deploy/vm/systemd/packetstation-worker.service

@@ -0,0 +1,67 @@
+# Packet Station: the worker (VISUAL feed, drawings, AD reconcile, integrity, backup check,
+# notifications, archive copies). Installed by deploy/vm/install.sh into /etc/systemd/system.
+# On production its first start is Spencer's approval of the scheduled VISUAL polling.
+[Unit]
+Description=Packet Station worker
+Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
+Requires=packetstation-migrate.service
+After=packetstation-migrate.service [email protected] network-online.target
+Wants=[email protected] network-online.target
+StartLimitIntervalSec=300
+StartLimitBurst=5
+
+[Service]
+Type=exec
+User=packetstation
+Group=packetstation
+EnvironmentFile=/etc/packetstation/packetstation.env
+LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
+LoadCredential=visualkit_password:/etc/packetstation/secrets/visualkit_password
+LoadCredential=mfileskit_password:/etc/packetstation/secrets/mfileskit_password
+LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
+LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
+ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh worker
+SyslogIdentifier=packetstation-worker
+# /run/packetstation holds the heartbeat file (WORKER_BEAT_FILE, set by ps-launch.sh).
+RuntimeDirectory=packetstation
+RuntimeDirectoryMode=0755
+Restart=on-failure
+RestartSec=10s
+TimeoutStopSec=60s
+WorkingDirectory=/
+UMask=0027
+MemoryMax=1G
+
+# Sandbox. Read-only file system except the assets and the archive mount. The backup folder
+# (BACKUP_DIR) is only read, so it needs no entry here.
+ReadWritePaths=/var/lib/packetstation/assets
+ReadWritePaths=-/mnt/packetstation-archive
+InaccessiblePaths=/etc/packetstation/secrets
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+# AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RemoveIPC=yes
+CapabilityBoundingSet=
+AmbientCapabilities=
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+# Not set: MemoryDenyWriteExecute (see packetstation-app.service).
+
+[Install]
+WantedBy=multi-user.target

deploy/vm/test.env.example

@@ -0,0 +1,101 @@
+# Packet Station test settings on the test VM (native, no Docker).
+# deploy/vm/install.sh --stack test copies this to /etc/packetstation/packetstation.env
+# (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
+# It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
+# The test VM never points at VISUAL production. It reads VISUAL DEV.
+#
+# systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
+# Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
+# Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
+
+# ---- Stack and site ----
+# prod, test, ci or dev. Guards against crossing stacks.
+STACK=test
+# The VISUAL site in every call. The docker-1 test stack runs with PESCO.
+SITE_ID=PESCO
+# Site time zone for schedules and display. The processes run in UTC.
+SITE_TZ=America/Denver
+# Public URL, used in links and mail.
+PUBLIC_URL=https://packetstation-test.pescoinc.biz
+# NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
+# opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
+TRUSTED_PROXY_HOST=192.168.101.12
+# DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
+LOG_LEVEL=INFO
+
+# ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
+# A folder means the Unix socket. PostgreSQL listens on no TCP port.
+PG_HOST=/var/run/postgresql
+PG_PORT=5432
+PG_DATABASE=packetstation
+PG_APP_USER=ps_app
+PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
+PG_MIGRATE_USER=ps_migrate
+PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
+
+# ---- Keys and folders ----
+# The services get the secrets as systemd credentials and ps-launch.sh points these at them.
+# The paths here are the files those credentials are loaded from.
+SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
+SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
+# Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
+ASSETS_DIR=/var/lib/packetstation/assets
+# Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
+BACKUP_DIR=/var/backups/packetstation
+# The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
+# `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
+# Admin on the Admin page, File locations.
+#ARCHIVE_ROOT=/mnt/packetstation-archive
+
+# ---- Uploads and storage ----
+UPLOAD_MAX_BYTES=26214400
+UPLOAD_MAX_BYTES_BOM=5242880
+ASSETS_QUOTA_WARN_PCT=85
+# 0 measures the file system under ASSETS_DIR.
+ASSETS_QUOTA_BYTES=0
+
+# ---- ClamAV (optional, off: Gitea issue 19) ----
+# Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
+#CLAMAV_HOST=127.0.0.1
+CLAMAV_PORT=3310
+CLAMAV_TIMEOUT_SECONDS=30
+
+# ---- LDAP (PESCO Active Directory) ----
+LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
+LDAP_BASE_DN=DC=PESCO,DC=com
+LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
+LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
+LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
+LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
+# The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
+LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
+LDAP_STARTTLS=false
+LDAP_ALLOW_INSECURE=false
+
+# ---- Mail (relay app-03, no auth) ----
+SMTP_HOST=app-03.pesco.com
+SMTP_PORT=25
+SMTP_FROM=[email protected]
+SMTP_STARTTLS=true
+SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
+# The test VM never mails people about test data. The Admin "Send a test" still sends.
+NOTIFY_DRY_RUN=true
+
+# ---- VisualKit (VISUAL ERP, read only) ----
+# VISUAL DEV. The test VM never points at production. Single quotes keep the backslash.
+VISUAL_SERVER='vsql.pesco.com\DEV'
+VISUAL_DATABASE=PESCOINCDEV
+VISUAL_USER=visualkit_ro
+VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
+# no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
+# store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
+VISUAL_ENCRYPT=no
+VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
+
+# ---- MFilesKit (M-Files, read only) ----
+MFILES_URL=https://mfiles.pescoinc.biz/REST
+MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
+MFILES_USERNAME=mfileskitRO
+# ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
+MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
+# Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.

deploy/vm/upgrade.sh

@@ -0,0 +1,280 @@
+#!/usr/bin/env bash
+# Install a Packet Station release on this VM and switch to it. Run by Spencer as root.
+# Installed as /usr/local/sbin/ps-upgrade by deploy/vm/install.sh.
+#
+# Usage:
+#   ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]
+#   ps-upgrade --rollback
+#
+# RELEASE.tar.gz comes from deploy/vm/make-release.sh. When RELEASE.tar.gz.sha256 lies next to it,
+# the archive is checked against it first.
+#
+# Steps:
+#   1. Unpack into /opt/packetstation/<build> and build its venv there: public pins from PyPI
+#      (server/requirements.txt), visualkit and mfileskit from the Gitea index with --index-url
+#      (server/requirements-pesco.txt), then the package itself with --no-deps; pip check.
+#      A release folder that is already complete is reused.
+#   2. A backup (ps-backup) when the database exists, unless --no-backup.
+#   3. Stop the app and the worker, point /opt/packetstation/current at the new release
+#      (previous keeps the old one), install its units and commands (install.sh --files-only).
+#   4. Run the migrations (restart packetstation-migrate.service). On failure: current goes back,
+#      the services stay stopped, and the restore command is printed.
+#   5. Start again whichever of the app and the worker ran before. A first install starts nothing
+#      unless --start: on production the first start is Spencer's approval of the VISUAL polling.
+#   6. Wait for /healthz, run ps-doctor (unless --no-doctor) and keep the newest 3 releases.
+#
+# --rollback points current back at previous and restarts what ran. Migrations are not undone:
+# roll back only to a release whose code works with the schema now in the database.
+#
+# Python: PYTHON (default python3.12) must be 3.12 or newer.
+set -euo pipefail
+umask 022
+
+OPT="${PS_OPT:-/opt/packetstation}"
+PYTHON="${PYTHON:-python3.12}"
+PESCO_INDEX_URL="${PESCO_INDEX_URL:-https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/}"
+PYPI_INDEX_URL="${PYPI_INDEX_URL:-https://pypi.org/simple/}"
+KEEP_RELEASES="${KEEP_RELEASES:-3}"
+HEALTH_URL="http://127.0.0.1:8000/healthz"
+SERVICES=(packetstation-app.service packetstation-worker.service)
+
+log() { printf 'ps-upgrade: %s\n' "$*" >&2; }
+die() { printf 'ps-upgrade: ERROR: %s\n' "$*" >&2; exit 1; }
+usage() { sed -n '2,31p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+
+tarball=""
+start=0
+backup=1
+doctor=1
+rollback=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --start) start=1; shift ;;
+    --no-backup) backup=0; shift ;;
+    --no-doctor) doctor=0; shift ;;
+    --rollback) rollback=1; shift ;;
+    -h|--help) usage ;;
+    -*) log "unknown option $1"; usage ;;
+    *) [[ -z "$tarball" ]] || usage; tarball="$1"; shift ;;
+  esac
+done
+
+[[ "$(id -u)" == "0" ]] || die "run as root"
+[[ $rollback -eq 1 || -n "$tarball" ]] || usage
+[[ "$KEEP_RELEASES" =~ ^[1-9][0-9]*$ ]] || die "KEEP_RELEASES must be a positive whole number"
+
+exec 9>/run/lock/packetstation-upgrade.lock
+flock -n 9 || die "another ps-upgrade is running"
+
+# ------------------------------------------------------------------ helpers
+
+was_active=()
+remember_active() {
+  local s
+  was_active=()
+  for s in "${SERVICES[@]}"; do
+    if systemctl is-active --quiet "$s"; then was_active+=("$s"); fi
+  done
+}
+
+switch_to() {
+  # Atomic: a new link next to current, then rename over it.
+  local target="$1"
+  ln -sfn -- "$target" "$OPT/.current.new"
+  mv -T -- "$OPT/.current.new" "$OPT/current"
+}
+
+wait_healthy() {
+  local i
+  for (( i = 0; i < 30; i++ )); do
+    if curl -fsS --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
+      log "the app answers $HEALTH_URL"
+      return 0
+    fi
+    sleep 2
+  done
+  log "WARNING: the app did not answer $HEALTH_URL within 60 s: journalctl -u packetstation-app -n 50"
+  return 1
+}
+
+start_again() {
+  local -a units=("$@")
+  if [[ ${#units[@]} -eq 0 ]]; then
+    log "the app and the worker were not running and stay stopped."
+    log "start them with: systemctl enable --now packetstation-app packetstation-worker"
+    return 0
+  fi
+  log "starting ${units[*]}"
+  systemctl start "${units[@]}"
+  local s
+  for s in "${units[@]}"; do
+    if [[ "$s" == packetstation-app.service ]]; then wait_healthy || true; fi
+  done
+}
+
+prune() {
+  local cur prev d
+  cur="$(readlink -f -- "$OPT/current" 2>/dev/null || true)"
+  prev="$(readlink -f -- "$OPT/previous" 2>/dev/null || true)"
+  local -a dirs=()
+  while IFS= read -r d; do dirs+=("$d"); done < <(
+    find "$OPT" -mindepth 1 -maxdepth 1 -type d -name '[A-Za-z0-9]*' -printf '%T@ %p\n' | sort -rn | cut -d' ' -f2-
+  )
+  # current and previous always stay; they count toward KEEP_RELEASES.
+  local allowed=$(( KEEP_RELEASES - 2 )) kept=0
+  for d in "${dirs[@]}"; do
+    if [[ "$d" == "$cur" || "$d" == "$prev" ]]; then continue; fi
+    kept=$(( kept + 1 ))
+    if [[ $kept -gt $allowed ]]; then
+      log "removing old release $(basename -- "$d")"
+      rm -rf -- "$d"
+    fi
+  done
+}
+
+# ------------------------------------------------------------------ rollback
+
+if [[ $rollback -eq 1 ]]; then
+  [[ -L "$OPT/previous" ]] || die "no previous release to go back to"
+  prev="$(readlink -f -- "$OPT/previous")"
+  cur="$(readlink -f -- "$OPT/current")"
+  [[ -f "$prev/.ready" ]] || die "previous release $prev is not complete"
+  remember_active
+  log "rolling back from $(basename -- "$cur") to $(basename -- "$prev")"
+  if [[ ${#was_active[@]} -gt 0 ]]; then systemctl stop "${was_active[@]}"; fi
+  switch_to "$prev"
+  ln -sfn -- "$cur" "$OPT/previous"
+  bash "$prev/deploy/vm/install.sh" --files-only
+  start_again "${was_active[@]}"
+  log "done. Migrations were not undone."
+  exit 0
+fi
+
+# ------------------------------------------------------------------ 1. unpack and build
+
+[[ -r "$tarball" ]] || die "cannot read $tarball"
+if [[ -r "$tarball.sha256" ]]; then
+  ( cd -- "$(dirname -- "$tarball")" && sha256sum --quiet -c -- "$(basename -- "$tarball").sha256" ) \
+    || die "$tarball does not match $tarball.sha256"
+  log "checksum ok"
+else
+  log "no $tarball.sha256 next to the archive; checksum not checked"
+fi
+
+# Every member must sit under one top folder packetstation-<build>/, with no absolute path and no "..".
+# The listing is read whole first: an early-exiting reader (head, grep -q) would make tar fail
+# with SIGPIPE under pipefail.
+members="$(tar -tzf "$tarball")" || die "cannot list $tarball"
+top="${members%%$'\n'*}"
+top="${top%%/*}"
+[[ "$top" =~ ^packetstation-([A-Za-z0-9][A-Za-z0-9._-]{0,63})$ ]] || die "unexpected top folder '$top' in $tarball"
+build="${BASH_REMATCH[1]}"
+outside="$(grep -Evc "^$top(/|$)" <<< "$members" || true)"
+[[ "$outside" == "0" ]] || die "$tarball has files outside $top/"
+dotdot="$(grep -Ec '(^|/)\.\.(/|$)' <<< "$members" || true)"
+[[ "$dotdot" == "0" ]] || die "$tarball has a '..' path"
+unset members
+
+dest="$OPT/$build"
+if [[ -f "$dest/.ready" ]]; then
+  log "release $build is already built in $dest"
+else
+  "$PYTHON" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)' \
+    || die "$PYTHON is not Python 3.12 or newer (set PYTHON=/path/to/python3.12)"
+  rm -rf -- "$dest"
+  install -d -o root -g root -m 0755 "$dest"
+  log "unpacking $build into $dest"
+  tar -xzf "$tarball" -C "$dest" --strip-components=1 --no-same-owner --no-same-permissions
+  [[ "$(head -n 1 "$dest/BUILD")" == "$build" ]] || die "BUILD in the archive does not say $build"
+  [[ -f "$dest/static/index.html" ]] || die "the archive has no built frontend (static/index.html)"
+
+  log "building the venv"
+  "$PYTHON" -m venv "$dest/venv"
+  pip=("$dest/venv/bin/python" -m pip --disable-pip-version-check --no-input)
+  # No pip.conf or PIP_* setting from the host may add an index.
+  export PIP_CONFIG_FILE=/dev/null
+  unset PIP_INDEX_URL PIP_EXTRA_INDEX_URL PIP_FIND_LINKS
+  "${pip[@]}" install --index-url "$PYPI_INDEX_URL" -r "$dest/server/requirements.txt"
+  # PESCO libraries from the Gitea index only. --index-url, never --extra-index-url.
+  "${pip[@]}" install --index-url "$PESCO_INDEX_URL" -r "$dest/server/requirements-pesco.txt"
+  # The package itself. The build backend (hatchling) comes from PyPI.
+  "${pip[@]}" install --index-url "$PYPI_INDEX_URL" --no-deps "$dest"
+  "${pip[@]}" check
+  # The release is read-only to the services. Owned by root, nothing writable by others.
+  chown -R root:root "$dest"
+  chmod -R u+rwX,go+rX,go-w "$dest"
+  # The units and ps-run start these directly; an archive made on Windows may have lost the x bit.
+  chmod 0755 "$dest"/deploy/vm/*.sh "$dest"/scripts/backup/*.sh
+  printf '%s\n' "$build" > "$dest/.ready"
+  log "release $build built"
+fi
+
+# ------------------------------------------------------------------ 2. backup
+
+first_install=1
+[[ -L "$OPT/current" ]] && first_install=0
+
+db_exists=0
+found="$(setpriv --reuid=postgres --regid=postgres --init-groups -- \
+  psql -X -At -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'packetstation'" 2>/dev/null || true)"
+if [[ "$found" == "1" ]]; then
+  db_exists=1
+fi
+if [[ $backup -eq 1 && $db_exists -eq 1 && $first_install -eq 0 ]]; then
+  log "backup before the upgrade"
+  /usr/local/sbin/ps-backup || die "the backup failed; nothing was changed. Fix it, or use --no-backup."
+fi
+
+# ------------------------------------------------------------------ 3. switch
+
+remember_active
+if [[ $start -eq 1 ]]; then was_active=("${SERVICES[@]}"); fi
+old=""
+if [[ $first_install -eq 0 ]]; then
+  old="$(readlink -f -- "$OPT/current")"
+fi
+if [[ "$old" == "$dest" ]]; then
+  log "release $build is already current"
+fi
+
+active_now=()
+for s in "${SERVICES[@]}"; do
+  if systemctl is-active --quiet "$s"; then active_now+=("$s"); fi
+done
+if [[ ${#active_now[@]} -gt 0 ]]; then
+  log "stopping ${active_now[*]}"
+  systemctl stop "${active_now[@]}"
+fi
+
+switch_to "$dest"
+if [[ -n "$old" && "$old" != "$dest" ]]; then
+  ln -sfn -- "$old" "$OPT/previous"
+fi
+log "current is $build"
+bash "$dest/deploy/vm/install.sh" --files-only
+
+# ------------------------------------------------------------------ 4. migrate
+
+log "running the migrations"
+if ! systemctl restart packetstation-migrate.service; then
+  journalctl -u packetstation-migrate.service -n 30 --no-pager >&2 || true
+  if [[ -n "$old" && "$old" != "$dest" ]]; then
+    switch_to "$old"
+    bash "$old/deploy/vm/install.sh" --files-only || true
+    log "current is back on $(basename -- "$old")"
+  fi
+  last="$(find /var/backups/packetstation -maxdepth 1 -name 'ps-*.dump' -printf '%f\n' 2>/dev/null | sort | tail -n 1)"
+  die "the migrations failed. The app and the worker are stopped. Read the log above. If the database
+must go back to the backup taken before this upgrade, restore it (docs/DEPLOY.md, \"Restore on a VM\"):
+  ps-restore --dump /var/backups/packetstation/${last:-<dump>} --db ps_restored"
+fi
+
+# ------------------------------------------------------------------ 5. and 6. start, check, prune
+
+start_again "${was_active[@]}"
+if [[ $doctor -eq 1 ]]; then
+  log "doctor"
+  /usr/local/sbin/ps-doctor || log "WARNING: the doctor reported a failure (see above)"
+fi
+prune
+log "done: $build"