Native VM install kit: systemd units, launcher, install and upgrade scripts; backups, restore and doctor without docker exec; DEPLOY.md VM, NPM and NFS sections
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
Executable
+510
@@ -0,0 +1,510 @@
|
||||
#!/usr/bin/env bash
|
||||
# Packet Station native VM install. Run by Spencer as root on a new test or production VM, from an
|
||||
# unpacked release (make-release.sh). Idempotent: run it again after a change and it only fixes
|
||||
# what differs. It never overwrites the env file or a secret, never starts the app or the worker,
|
||||
# and never touches VISUAL, M-Files or AD.
|
||||
#
|
||||
# Usage:
|
||||
# install.sh --stack test|prod [--ca-bundle FILE] [--generate-secrets] [--skip-packages]
|
||||
# install.sh --files-only
|
||||
#
|
||||
# --stack S test or prod. Picks the env file example. Read from the env file when it exists.
|
||||
# --ca-bundle FILE the PESCO CA chain (PEM, scripts/Export-PescoCaBundle.ps1). Installed as
|
||||
# /etc/packetstation/ca/pesco-ca.pem and into the system trust store.
|
||||
# --generate-secrets create the four internal secrets that are missing (both PostgreSQL role
|
||||
# passwords, session_key, seal_key) from /dev/urandom. Never replaces a file.
|
||||
# --skip-packages leave apt alone (repositories and packages already in place).
|
||||
# --files-only only the systemd units, the ps-* commands, backup.conf and the firewall
|
||||
# table. ps-upgrade runs this from the new release.
|
||||
#
|
||||
# What it does, in order:
|
||||
# 1. apt: PostgreSQL 16 (PGDG), ODBC Driver 18 (Microsoft), Python 3.12, nftables, nfs-common.
|
||||
# 2. User and group packetstation, uid and gid 10001 (the image's uid, and the NFS share's).
|
||||
# 3. Folders: /etc/packetstation, /var/lib/packetstation/assets, /var/backups/packetstation,
|
||||
# /opt/packetstation, the archive mount point /mnt/packetstation-archive.
|
||||
# 4. /etc/packetstation/packetstation.env from the stack's example, if missing.
|
||||
# 5. The PESCO CA, when --ca-bundle is given.
|
||||
# 6. Secret files: owner root:packetstation, mode 0440, one line, no carriage return.
|
||||
# 7. PostgreSQL: socket only (conf.d/packetstation.conf), scram for ps_app and ps_migrate on the
|
||||
# socket (pg_hba.conf), the roles and the database as deploy/postgres/initdb makes them.
|
||||
# 8. Units, commands, backup timer, firewall (the files-only part).
|
||||
set -euo pipefail
|
||||
umask 022
|
||||
|
||||
PS_USER=packetstation
|
||||
PS_ID=10001
|
||||
PG_VERSION=16
|
||||
PG_CLUSTER=main
|
||||
ETC=/etc/packetstation
|
||||
SECRETS="$ETC/secrets"
|
||||
ENV_FILE="$ETC/packetstation.env"
|
||||
CA_FILE="$ETC/ca/pesco-ca.pem"
|
||||
NFT_FILE="$ETC/packetstation.nft"
|
||||
BACKUP_CONF="$ETC/backup.conf"
|
||||
ASSETS=/var/lib/packetstation/assets
|
||||
BACKUPS=/var/backups/packetstation
|
||||
ARCHIVE_MOUNT=/mnt/packetstation-archive
|
||||
OPT=/opt/packetstation
|
||||
SBIN=/usr/local/sbin
|
||||
UNIT_DIR=/etc/systemd/system
|
||||
PG_ETC="/etc/postgresql/$PG_VERSION/$PG_CLUSTER"
|
||||
PG_UNIT="postgresql@$PG_VERSION-$PG_CLUSTER.service"
|
||||
PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin"
|
||||
DB_LOCALE=en_US.UTF-8
|
||||
|
||||
# Signing keys, checked by fingerprint before apt trusts them.
|
||||
PGDG_KEY_URL=https://www.postgresql.org/media/keys/ACCC4CF8.asc
|
||||
PGDG_KEY_FPR=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
|
||||
MS_KEY_URL=https://packages.microsoft.com/keys/microsoft.asc
|
||||
MS_KEY_FPR=BC528686B50D79E339D3721CEB3E94ADBE1229CF
|
||||
|
||||
ALL_SECRETS=(postgres_migrate_password postgres_app_password visualkit_password mfileskit_password
|
||||
ldap_bind_password session_key seal_key)
|
||||
GENERATED_SECRETS=(postgres_migrate_password postgres_app_password session_key seal_key)
|
||||
|
||||
here="$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd -P)"
|
||||
release="$(cd -- "$here/../.." && pwd -P)"
|
||||
|
||||
log() { printf 'install: %s\n' "$*" >&2; }
|
||||
die() { printf 'install: ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
usage() { sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
|
||||
stack=""
|
||||
ca_bundle=""
|
||||
generate=0
|
||||
skip_packages=0
|
||||
files_only=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--stack) [[ $# -ge 2 ]] || usage; stack="$2"; shift 2 ;;
|
||||
--ca-bundle) [[ $# -ge 2 ]] || usage; ca_bundle="$2"; shift 2 ;;
|
||||
--generate-secrets) generate=1; shift ;;
|
||||
--skip-packages) skip_packages=1; shift ;;
|
||||
--files-only) files_only=1; shift ;;
|
||||
-h|--help) usage ;;
|
||||
*) log "unknown argument $1"; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$(id -u)" == "0" ]] || die "run as root"
|
||||
[[ -z "$stack" || "$stack" == "test" || "$stack" == "prod" ]] || die "--stack must be test or prod"
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
# A KEY=value from the env file, with surrounding quotes removed. Empty when unset.
|
||||
env_value() {
|
||||
local key="$1" line value
|
||||
[[ -r "$ENV_FILE" ]] || return 0
|
||||
line="$(grep -E "^[[:space:]]*$key=" "$ENV_FILE" | tail -n 1 || true)"
|
||||
value="${line#*=}"
|
||||
value="${value%$'\r'}"
|
||||
value="${value#\"}"; value="${value%\"}"
|
||||
value="${value#\'}"; value="${value%\'}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
# Install a text file only when it differs. Strips carriage returns (a Windows checkout).
|
||||
# Prints "changed" when it wrote the file.
|
||||
put_file() {
|
||||
local src="$1" dst="$2" owner="$3" mode="$4" tmp
|
||||
tmp="$(mktemp)"
|
||||
sed 's/\r$//' -- "$src" > "$tmp"
|
||||
if [[ -f "$dst" ]] && cmp -s -- "$tmp" "$dst"; then
|
||||
chown "$owner" -- "$dst"; chmod "$mode" -- "$dst"
|
||||
rm -f -- "$tmp"
|
||||
return 0
|
||||
fi
|
||||
install -o "${owner%%:*}" -g "${owner##*:}" -m "$mode" -- "$tmp" "$dst"
|
||||
rm -f -- "$tmp"
|
||||
echo changed
|
||||
}
|
||||
|
||||
as_postgres() {
|
||||
setpriv --reuid=postgres --regid=postgres --init-groups -- "$@"
|
||||
}
|
||||
|
||||
is_ipv4() { [[ "$1" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; }
|
||||
is_ipv6() { [[ "$1" =~ ^[0-9A-Fa-f:]+$ && "$1" == *:* ]]; }
|
||||
|
||||
# ------------------------------------------------------------------ 1. packages
|
||||
|
||||
os_id=""
|
||||
os_version=""
|
||||
os_codename=""
|
||||
read_os() {
|
||||
# shellcheck disable=SC1091
|
||||
os_id="$(. /etc/os-release && printf '%s' "${ID:-}")"
|
||||
# shellcheck disable=SC1091
|
||||
os_version="$(. /etc/os-release && printf '%s' "${VERSION_ID:-}")"
|
||||
# shellcheck disable=SC1091
|
||||
os_codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")"
|
||||
}
|
||||
|
||||
fetch_key() {
|
||||
local url="$1" fpr="$2" dst="$3" tmp found
|
||||
tmp="$(mktemp -d)"
|
||||
curl -fsSL --proto '=https' "$url" -o "$tmp/key.asc"
|
||||
found="$(gpg --batch --show-keys --with-colons "$tmp/key.asc" 2>/dev/null | awk -F: '$1 == "fpr" && !seen { print $10; seen = 1 }')"
|
||||
if [[ "$found" != "$fpr" ]]; then
|
||||
rm -rf -- "$tmp"
|
||||
die "the key from $url has fingerprint '${found:-none}', expected $fpr. Check it before going on."
|
||||
fi
|
||||
gpg --batch --yes --dearmor -o "$tmp/key.gpg" "$tmp/key.asc"
|
||||
install -o root -g root -m 0644 "$tmp/key.gpg" "$dst"
|
||||
rm -rf -- "$tmp"
|
||||
}
|
||||
|
||||
step_packages() {
|
||||
local ms_repo python_pkgs=()
|
||||
case "$os_id $os_version" in
|
||||
"ubuntu 24.04")
|
||||
ms_repo="https://packages.microsoft.com/ubuntu/24.04/prod $os_codename main"
|
||||
python_pkgs=(python3.12 python3.12-venv) ;;
|
||||
"debian 12")
|
||||
ms_repo="https://packages.microsoft.com/debian/12/prod $os_codename main"
|
||||
log "Debian 12 ships Python 3.11. Packet Station needs 3.12: give ps-upgrade PYTHON=/path/to/python3.12." ;;
|
||||
*) die "unsupported OS '$os_id $os_version'. Supported: Ubuntu 24.04, Debian 12." ;;
|
||||
esac
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "apt: base packages"
|
||||
apt-get update -q
|
||||
apt-get install -y -q --no-install-recommends \
|
||||
ca-certificates curl gnupg locales tzdata nftables nfs-common util-linux coreutils findutils
|
||||
|
||||
# The database locale. Generated before PostgreSQL so its first cluster knows it too.
|
||||
if ! locale -a 2>/dev/null | grep -qiE '^en_US\.utf-?8$'; then
|
||||
log "generating locale $DB_LOCALE"
|
||||
if [[ -f /etc/locale.gen ]]; then
|
||||
sed -i 's/^# *\(en_US.UTF-8 UTF-8\)/\1/' /etc/locale.gen
|
||||
fi
|
||||
locale-gen en_US.UTF-8 >/dev/null
|
||||
fi
|
||||
|
||||
log "apt: PGDG and Microsoft repositories"
|
||||
fetch_key "$PGDG_KEY_URL" "$PGDG_KEY_FPR" /usr/share/keyrings/pgdg.gpg
|
||||
fetch_key "$MS_KEY_URL" "$MS_KEY_FPR" /usr/share/keyrings/microsoft-prod.gpg
|
||||
printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/pgdg.gpg] https://apt.postgresql.org/pub/repos/apt %s-pgdg main\n' \
|
||||
"$os_codename" > /etc/apt/sources.list.d/pgdg.list
|
||||
printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] %s\n' "$ms_repo" \
|
||||
> /etc/apt/sources.list.d/microsoft-prod.list
|
||||
apt-get update -q
|
||||
|
||||
log "apt: PostgreSQL $PG_VERSION, ODBC Driver 18, Python"
|
||||
ACCEPT_EULA=Y apt-get install -y -q --no-install-recommends \
|
||||
"postgresql-$PG_VERSION" "postgresql-client-$PG_VERSION" msodbcsql18 unixodbc "${python_pkgs[@]}"
|
||||
odbcinst -q -d -n "ODBC Driver 18 for SQL Server" >/dev/null \
|
||||
|| die "ODBC Driver 18 for SQL Server is not registered with unixODBC"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 2. user
|
||||
|
||||
step_user() {
|
||||
local g u
|
||||
g="$(getent group "$PS_USER" | cut -d: -f3 || true)"
|
||||
if [[ -z "$g" ]]; then
|
||||
if getent group "$PS_ID" >/dev/null; then
|
||||
die "gid $PS_ID is taken by group $(getent group "$PS_ID" | cut -d: -f1)"
|
||||
fi
|
||||
groupadd --system --gid "$PS_ID" "$PS_USER"
|
||||
log "created group $PS_USER ($PS_ID)"
|
||||
elif [[ "$g" != "$PS_ID" ]]; then
|
||||
die "group $PS_USER has gid $g, expected $PS_ID (the NFS share and the backups expect $PS_ID)"
|
||||
fi
|
||||
u="$(id -u "$PS_USER" 2>/dev/null || true)"
|
||||
if [[ -z "$u" ]]; then
|
||||
if getent passwd "$PS_ID" >/dev/null; then
|
||||
die "uid $PS_ID is taken by user $(getent passwd "$PS_ID" | cut -d: -f1)"
|
||||
fi
|
||||
useradd --system --uid "$PS_ID" --gid "$PS_ID" --home-dir /var/lib/packetstation --no-create-home \
|
||||
--shell /usr/sbin/nologin --comment "Packet Station" "$PS_USER"
|
||||
log "created user $PS_USER ($PS_ID)"
|
||||
elif [[ "$u" != "$PS_ID" ]]; then
|
||||
die "user $PS_USER has uid $u, expected $PS_ID"
|
||||
fi
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 3. folders
|
||||
|
||||
step_dirs() {
|
||||
install -d -o root -g "$PS_USER" -m 0750 "$ETC" "$SECRETS"
|
||||
install -d -o root -g root -m 0755 "$ETC/ca" "$OPT"
|
||||
install -d -o root -g "$PS_USER" -m 0750 /var/lib/packetstation "$BACKUPS"
|
||||
install -d -o "$PS_USER" -g "$PS_USER" -m 0750 "$ASSETS"
|
||||
# The mount point stays root's and read-only to packetstation, so while the share is not mounted
|
||||
# nothing can be written into the VM's own disk by mistake.
|
||||
if mountpoint -q "$ARCHIVE_MOUNT" 2>/dev/null; then
|
||||
log "$ARCHIVE_MOUNT is mounted; left as it is"
|
||||
else
|
||||
install -d -o root -g root -m 0755 "$ARCHIVE_MOUNT"
|
||||
fi
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 4. env file
|
||||
|
||||
step_env() {
|
||||
local example have
|
||||
if [[ -e "$ENV_FILE" ]]; then
|
||||
have="$(env_value STACK)"
|
||||
if [[ -n "$stack" && "$have" != "$stack" ]]; then
|
||||
die "$ENV_FILE says STACK=$have, but --stack $stack was given"
|
||||
fi
|
||||
chown root:"$PS_USER" "$ENV_FILE"; chmod 0640 "$ENV_FILE"
|
||||
log "kept $ENV_FILE (STACK=$have)"
|
||||
return 0
|
||||
fi
|
||||
[[ -n "$stack" ]] || die "no $ENV_FILE yet: give --stack test or --stack prod"
|
||||
example="$here/$stack.env.example"
|
||||
[[ -r "$example" ]] || die "missing $example"
|
||||
put_file "$example" "$ENV_FILE" "root:$PS_USER" 0640 >/dev/null
|
||||
log "wrote $ENV_FILE from $(basename -- "$example"). Read it through before the first start."
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 5. CA
|
||||
|
||||
step_ca() {
|
||||
if [[ -z "$ca_bundle" ]]; then
|
||||
[[ -f "$CA_FILE" ]] || log "no PESCO CA yet: run again with --ca-bundle FILE (LDAP and SMTP need it)"
|
||||
return 0
|
||||
fi
|
||||
[[ -r "$ca_bundle" ]] || die "cannot read $ca_bundle"
|
||||
grep -q 'BEGIN CERTIFICATE' "$ca_bundle" || die "$ca_bundle holds no PEM certificate"
|
||||
if grep -q 'PRIVATE KEY' "$ca_bundle"; then
|
||||
die "$ca_bundle holds a private key. Export only the certificates."
|
||||
fi
|
||||
put_file "$ca_bundle" "$CA_FILE" root:root 0644 >/dev/null
|
||||
# update-ca-certificates wants one certificate per .crt file.
|
||||
local dir=/usr/local/share/ca-certificates/pesco tmp
|
||||
tmp="$(mktemp -d)"
|
||||
sed 's/\r$//' -- "$CA_FILE" | awk -v d="$tmp" '
|
||||
/-----BEGIN CERTIFICATE-----/ { n++; f = sprintf("%s/pesco-ca-%02d.crt", d, n) }
|
||||
f { print > f }
|
||||
/-----END CERTIFICATE-----/ { close(f); f = "" }'
|
||||
rm -rf -- "$dir"
|
||||
install -d -o root -g root -m 0755 "$dir"
|
||||
install -o root -g root -m 0644 "$tmp"/*.crt "$dir"/
|
||||
rm -rf -- "$tmp"
|
||||
update-ca-certificates >/dev/null
|
||||
log "PESCO CA installed: $CA_FILE and the system trust store ($(find "$dir" -name '*.crt' | wc -l) certificates)"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 6. secrets
|
||||
|
||||
new_secret() {
|
||||
# 32 random bytes as 64 hex characters. Held in a variable and written by a builtin, never on argv.
|
||||
local name="$1" value tmp
|
||||
value="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')"
|
||||
[[ ${#value} -eq 64 ]] || die "could not read /dev/urandom"
|
||||
tmp="$(mktemp "$SECRETS/.$name.XXXXXX")"
|
||||
printf '%s' "$value" > "$tmp"
|
||||
unset value
|
||||
chown root:"$PS_USER" "$tmp"; chmod 0440 "$tmp"
|
||||
mv -- "$tmp" "$SECRETS/$name"
|
||||
log "generated secret $name"
|
||||
}
|
||||
|
||||
missing_secrets=()
|
||||
step_secrets() {
|
||||
local name f g
|
||||
missing_secrets=()
|
||||
for name in "${ALL_SECRETS[@]}"; do
|
||||
f="$SECRETS/$name"
|
||||
if [[ ! -e "$f" && $generate -eq 1 ]]; then
|
||||
for g in "${GENERATED_SECRETS[@]}"; do
|
||||
if [[ "$g" == "$name" ]]; then new_secret "$name"; fi
|
||||
done
|
||||
fi
|
||||
if [[ ! -e "$f" ]]; then
|
||||
missing_secrets+=("$name")
|
||||
continue
|
||||
fi
|
||||
chown root:"$PS_USER" -- "$f"; chmod 0440 -- "$f"
|
||||
[[ -s "$f" ]] || die "secret $name is empty"
|
||||
if [[ "$(wc -l < "$f")" -gt 1 ]] || grep -q $'\r' -- "$f"; then
|
||||
die "secret $name must be one line with no carriage return"
|
||||
fi
|
||||
done
|
||||
if [[ ${#missing_secrets[@]} -gt 0 ]]; then
|
||||
log "secrets still missing in $SECRETS: ${missing_secrets[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
have_secret() { [[ -s "$SECRETS/$1" ]]; }
|
||||
|
||||
# ------------------------------------------------------------------ 7. PostgreSQL
|
||||
|
||||
step_postgres() {
|
||||
local changed="" hba="$PG_ETC/pg_hba.conf" tmp
|
||||
[[ -d "$PG_ETC" ]] || die "no PostgreSQL cluster $PG_VERSION/$PG_CLUSTER in $PG_ETC"
|
||||
grep -Eq "^[[:space:]]*include_dir[[:space:]]*=[[:space:]]*'conf\.d'" "$PG_ETC/postgresql.conf" \
|
||||
|| die "$PG_ETC/postgresql.conf has no include_dir = 'conf.d'"
|
||||
install -d -o postgres -g postgres -m 0755 "$PG_ETC/conf.d"
|
||||
changed="$(put_file "$here/postgres/packetstation.conf" "$PG_ETC/conf.d/packetstation.conf" postgres:postgres 0644)"
|
||||
|
||||
# ps_app and ps_migrate log in on the socket with their passwords (scram). The block goes first,
|
||||
# before Debian's "local all all peer", which would refuse them (the OS user is packetstation).
|
||||
if ! grep -q '^# packetstation begin' "$hba"; then
|
||||
tmp="$(mktemp)"
|
||||
awk '
|
||||
!done && $0 !~ /^[[:space:]]*(#|$)/ {
|
||||
print "# packetstation begin (deploy/vm/install.sh)"
|
||||
print "local packetstation ps_app,ps_migrate scram-sha-256"
|
||||
print "# packetstation end"
|
||||
print ""
|
||||
done = 1
|
||||
}
|
||||
{ print }' "$hba" > "$tmp"
|
||||
install -o postgres -g postgres -m 0640 "$tmp" "$hba"
|
||||
rm -f -- "$tmp"
|
||||
log "pg_hba.conf: ps_app and ps_migrate use scram-sha-256 on the socket"
|
||||
[[ -n "$changed" ]] || changed=reload
|
||||
fi
|
||||
|
||||
systemctl enable --quiet "$PG_UNIT" 2>/dev/null || true
|
||||
if [[ "$changed" == "changed" ]]; then
|
||||
log "restarting PostgreSQL (conf.d/packetstation.conf changed)"
|
||||
systemctl restart "$PG_UNIT"
|
||||
elif [[ "$changed" == "reload" ]]; then
|
||||
systemctl start "$PG_UNIT"
|
||||
systemctl reload "$PG_UNIT"
|
||||
else
|
||||
systemctl start "$PG_UNIT"
|
||||
fi
|
||||
|
||||
if ! have_secret postgres_migrate_password || ! have_secret postgres_app_password; then
|
||||
log "skipping roles and database: the PostgreSQL secrets are missing (or use --generate-secrets)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "PostgreSQL roles ps_migrate and ps_app, database packetstation"
|
||||
# The passwords reach psql through its environment (\getenv), never through argv or a file.
|
||||
# A failed statement is not logged with its text (log_min_error_statement), so a password
|
||||
# cannot land in the server log.
|
||||
PS_MIGRATE_PW="$(<"$SECRETS/postgres_migrate_password")"
|
||||
PS_APP_PW="$(<"$SECRETS/postgres_app_password")"
|
||||
export PS_MIGRATE_PW PS_APP_PW
|
||||
as_postgres "$PG_BIN/psql" -X -q -v ON_ERROR_STOP=1 -d postgres <<SQL
|
||||
SET log_min_error_statement = panic;
|
||||
SET log_statement = 'none';
|
||||
\\getenv migrate_pw PS_MIGRATE_PW
|
||||
\\getenv app_pw PS_APP_PW
|
||||
SELECT 'CREATE ROLE ps_migrate' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_migrate') \\gexec
|
||||
SELECT 'CREATE ROLE ps_app' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_app') \\gexec
|
||||
ALTER ROLE ps_migrate WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'migrate_pw';
|
||||
ALTER ROLE ps_app WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'app_pw';
|
||||
\\unset migrate_pw
|
||||
\\unset app_pw
|
||||
SELECT format('CREATE DATABASE packetstation OWNER ps_migrate ENCODING %L LOCALE %L TEMPLATE template0',
|
||||
'UTF8', '$DB_LOCALE')
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'packetstation') \\gexec
|
||||
REVOKE ALL ON DATABASE packetstation FROM PUBLIC;
|
||||
GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app;
|
||||
\\connect packetstation
|
||||
REVOKE ALL ON SCHEMA public FROM PUBLIC;
|
||||
ALTER SCHEMA public OWNER TO ps_migrate;
|
||||
SQL
|
||||
unset PS_MIGRATE_PW PS_APP_PW
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 8. files
|
||||
|
||||
write_firewall() {
|
||||
local proxy rule tmp
|
||||
proxy="$(env_value TRUSTED_PROXY_HOST)"
|
||||
if is_ipv4 "$proxy"; then
|
||||
rule="ip saddr $proxy"
|
||||
elif is_ipv6 "$proxy"; then
|
||||
rule="ip6 saddr $proxy"
|
||||
else
|
||||
die "TRUSTED_PROXY_HOST in $ENV_FILE must be NPM's IP address (it is '${proxy}')"
|
||||
fi
|
||||
tmp="$(mktemp)"
|
||||
cat > "$tmp" <<NFT
|
||||
#!/usr/sbin/nft -f
|
||||
# Written by deploy/vm/install.sh from TRUSTED_PROXY_HOST in $ENV_FILE. Do not edit:
|
||||
# change the env file and run install.sh --files-only. Loaded by packetstation-firewall.service.
|
||||
# Port 8000 (the app) answers only NPM on docker-1 and the VM itself. Nothing else is filtered here.
|
||||
table inet packetstation
|
||||
delete table inet packetstation
|
||||
table inet packetstation {
|
||||
chain input {
|
||||
type filter hook input priority filter; policy accept;
|
||||
tcp dport 8000 iifname "lo" accept
|
||||
tcp dport 8000 $rule accept
|
||||
tcp dport 8000 counter drop
|
||||
}
|
||||
}
|
||||
NFT
|
||||
if command -v nft >/dev/null 2>&1; then
|
||||
nft -c -f "$tmp" || { rm -f -- "$tmp"; die "nft refused the generated table"; }
|
||||
fi
|
||||
put_file "$tmp" "$NFT_FILE" root:root 0644 >/dev/null
|
||||
rm -f -- "$tmp"
|
||||
}
|
||||
|
||||
step_files() {
|
||||
local f st
|
||||
[[ -r "$ENV_FILE" ]] || die "no $ENV_FILE: run install.sh --stack test|prod first"
|
||||
st="$(env_value STACK)"
|
||||
[[ "$st" == "test" || "$st" == "prod" ]] || die "STACK in $ENV_FILE must be test or prod"
|
||||
|
||||
for f in "$here"/systemd/*.service; do
|
||||
put_file "$f" "$UNIT_DIR/$(basename -- "$f")" root:root 0644 >/dev/null
|
||||
done
|
||||
put_file "$release/scripts/backup/packetstation-backup.service" "$UNIT_DIR/packetstation-backup.service" root:root 0644 >/dev/null
|
||||
put_file "$release/scripts/backup/packetstation-backup.timer" "$UNIT_DIR/packetstation-backup.timer" root:root 0644 >/dev/null
|
||||
|
||||
put_file "$here/ps-run.sh" "$SBIN/ps-run" root:root 0755 >/dev/null
|
||||
put_file "$here/ps-doctor.sh" "$SBIN/ps-doctor" root:root 0755 >/dev/null
|
||||
put_file "$here/upgrade.sh" "$SBIN/ps-upgrade" root:root 0755 >/dev/null
|
||||
put_file "$release/scripts/backup/ps-backup.sh" "$SBIN/ps-backup" root:root 0755 >/dev/null
|
||||
put_file "$release/scripts/backup/ps-restore.sh" "$SBIN/ps-restore" root:root 0755 >/dev/null
|
||||
|
||||
if [[ ! -e "$BACKUP_CONF" ]]; then
|
||||
printf '# name database folder\n%s packetstation %s\n' "$st" "$BACKUPS" > "$BACKUP_CONF"
|
||||
chmod 0644 "$BACKUP_CONF"
|
||||
log "wrote $BACKUP_CONF"
|
||||
fi
|
||||
|
||||
write_firewall
|
||||
systemctl daemon-reload
|
||||
systemctl enable --quiet packetstation-firewall.service packetstation-backup.timer
|
||||
if systemctl is-active --quiet packetstation-firewall.service; then
|
||||
systemctl reload packetstation-firewall.service
|
||||
else
|
||||
systemctl start packetstation-firewall.service
|
||||
fi
|
||||
systemctl start packetstation-backup.timer
|
||||
log "units, ps-run, ps-doctor, ps-upgrade, ps-backup, ps-restore and the firewall table are in place"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ main
|
||||
|
||||
if [[ $files_only -eq 1 ]]; then
|
||||
step_files
|
||||
exit 0
|
||||
fi
|
||||
|
||||
read_os
|
||||
if [[ $skip_packages -eq 0 ]]; then
|
||||
step_packages
|
||||
fi
|
||||
for tool in setpriv systemctl install mountpoint; do
|
||||
command -v "$tool" >/dev/null 2>&1 || die "$tool is not installed"
|
||||
done
|
||||
step_user
|
||||
step_dirs
|
||||
step_env
|
||||
step_ca
|
||||
step_secrets
|
||||
step_postgres
|
||||
step_files
|
||||
|
||||
cat >&2 <<EOF
|
||||
install: done for STACK=$(env_value STACK).
|
||||
install: next:
|
||||
- read $ENV_FILE through
|
||||
- secrets still missing: ${missing_secrets[*]:-none}
|
||||
- install a release: ps-upgrade /path/to/packetstation-<build>.tar.gz
|
||||
- the app and the worker are not enabled or started. Starting them is a separate step (docs/DEPLOY.md).
|
||||
EOF
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a release archive for the native VMs. Runs on a build host as a normal user, inside a git
|
||||
# checkout of this repository: docker-1 (with --image) or a PC with Node (with --npm). Never on a VM.
|
||||
#
|
||||
# Usage:
|
||||
# make-release.sh --image gt.pescoinc.biz/spencerm/packetstation:sha-<sha> [--out DIR]
|
||||
# make-release.sh --npm [--ref REF] [--build NAME] [--out DIR]
|
||||
#
|
||||
# --image IMAGE take the built frontend out of a CI-tested image (docker create and docker cp,
|
||||
# nothing runs). The build name and the commit come from the image's build label,
|
||||
# so the source and the frontend are the tested commit.
|
||||
# --npm build the frontend here with npm ci and npm run build, from the same commit.
|
||||
# --ref REF the commit for --npm (default HEAD). Uncommitted changes are never included.
|
||||
# --build NAME the build name for --npm (default sha-<full sha>, as CI names images).
|
||||
# --out DIR where to write the archive (default the current folder).
|
||||
#
|
||||
# Writes packetstation-<build>.tar.gz and packetstation-<build>.tar.gz.sha256. The archive holds one
|
||||
# folder, packetstation-<build>/, with BUILD, pyproject.toml, README.md, server/, deploy/, scripts/,
|
||||
# docs/ and static/ (the built frontend). Python packages are not in it: ps-upgrade builds the venv
|
||||
# on the VM from the pinned requirements.
|
||||
set -euo pipefail
|
||||
umask 022
|
||||
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
LABEL=biz.pescoinc.packetstation.build
|
||||
|
||||
log() { printf 'make-release: %s\n' "$*" >&2; }
|
||||
die() { printf 'make-release: ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
usage() { sed -n '2,21p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
|
||||
image=""
|
||||
npm=0
|
||||
ref=""
|
||||
build=""
|
||||
out="."
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--image) [[ $# -ge 2 ]] || usage; image="$2"; shift 2 ;;
|
||||
--npm) npm=1; shift ;;
|
||||
--ref) [[ $# -ge 2 ]] || usage; ref="$2"; shift 2 ;;
|
||||
--build) [[ $# -ge 2 ]] || usage; build="$2"; shift 2 ;;
|
||||
--out) [[ $# -ge 2 ]] || usage; out="$2"; shift 2 ;;
|
||||
-h|--help) usage ;;
|
||||
*) log "unknown argument $1"; usage ;;
|
||||
esac
|
||||
done
|
||||
if [[ -n "$image" && $npm -eq 1 ]] || [[ -z "$image" && $npm -eq 0 ]]; then
|
||||
die "give exactly one of --image or --npm"
|
||||
fi
|
||||
if [[ -n "$image" && ( -n "$ref" || -n "$build" ) ]]; then
|
||||
die "--ref and --build come from the image's label with --image"
|
||||
fi
|
||||
[[ "$image" != *:latest && ( -z "$image" || "$image" == *:* ) ]] || die "pull an image by its tag, never latest"
|
||||
|
||||
repo="$(git rev-parse --show-toplevel 2>/dev/null)" || die "run inside a git checkout of Packet Station"
|
||||
cd -- "$repo"
|
||||
|
||||
if [[ -n "$image" ]]; then
|
||||
"$DOCKER" image inspect "$image" >/dev/null 2>&1 || "$DOCKER" pull "$image" >/dev/null
|
||||
build="$("$DOCKER" image inspect -f "{{ index .Config.Labels \"$LABEL\" }}" "$image")"
|
||||
[[ "$build" =~ ^sha-[0-9a-f]{7,40}$ ]] || die "image label $LABEL is '$build', expected sha-<commit>"
|
||||
ref="${build#sha-}"
|
||||
else
|
||||
ref="${ref:-HEAD}"
|
||||
fi
|
||||
sha="$(git rev-parse --verify --quiet "$ref^{commit}")" || die "commit $ref is not in this checkout (git fetch first)"
|
||||
if [[ -z "$build" ]]; then
|
||||
build="sha-$sha"
|
||||
fi
|
||||
[[ "$build" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] || die "build name '$build' must be letters, digits, '.', '_' or '-'"
|
||||
|
||||
name="packetstation-$build"
|
||||
mkdir -p -- "$out"
|
||||
out="$(cd -- "$out" && pwd)"
|
||||
work="$(mktemp -d)"
|
||||
cid=""
|
||||
cleanup() {
|
||||
if [[ -n "$cid" ]]; then "$DOCKER" rm -f "$cid" >/dev/null 2>&1 || true; fi
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log "source of $sha"
|
||||
# LF line endings whatever the checkout's autocrlf: these files run on Linux.
|
||||
git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix="$name/" "$sha" \
|
||||
pyproject.toml README.md server deploy scripts docs | tar -xf - -C "$work"
|
||||
printf '%s\n' "$build" > "$work/$name/BUILD"
|
||||
[[ -f "$work/$name/deploy/vm/ps-launch.sh" ]] || die "commit $sha has no deploy/vm (older than phase 10)"
|
||||
chmod 0755 "$work/$name"/deploy/vm/*.sh "$work/$name"/scripts/backup/*.sh
|
||||
|
||||
if [[ -n "$image" ]]; then
|
||||
log "frontend from $image"
|
||||
cid="$("$DOCKER" create "$image")"
|
||||
"$DOCKER" cp "$cid:/app/static" "$work/$name/static"
|
||||
else
|
||||
log "frontend: npm ci and npm run build at $sha"
|
||||
git -c core.autocrlf=false -c core.eol=lf archive --format=tar --prefix=frontend/ "$sha:frontend" \
|
||||
| tar -xf - -C "$work"
|
||||
( cd -- "$work/frontend" && npm ci --no-audit --no-fund && npm run build ) >&2
|
||||
cp -R -- "$work/frontend/dist" "$work/$name/static"
|
||||
fi
|
||||
[[ -f "$work/$name/static/index.html" ]] || die "no static/index.html in the built frontend"
|
||||
|
||||
log "writing $out/$name.tar.gz"
|
||||
tar -czf "$out/$name.tar.gz" -C "$work" --owner=0 --group=0 --numeric-owner --sort=name "$name"
|
||||
( cd -- "$out" && sha256sum -- "$name.tar.gz" > "$name.tar.gz.sha256" )
|
||||
log "done: $out/$name.tar.gz ($(wc -c < "$out/$name.tar.gz") bytes) and its .sha256"
|
||||
@@ -0,0 +1,13 @@
|
||||
# Packet Station on this VM. Installed by deploy/vm/install.sh as
|
||||
# /etc/postgresql/16/main/conf.d/packetstation.conf. A change here needs a PostgreSQL restart.
|
||||
|
||||
# Local socket only. The app, the worker, migrate and the backups all connect through
|
||||
# /var/run/postgresql. Nothing on the network can reach PostgreSQL.
|
||||
listen_addresses = ''
|
||||
|
||||
# The same as the postgres container on docker-1: the server clock zone is UTC.
|
||||
# The app shows times in SITE_TZ.
|
||||
timezone = 'UTC'
|
||||
log_timezone = 'UTC'
|
||||
|
||||
password_encryption = 'scram-sha-256'
|
||||
@@ -0,0 +1,102 @@
|
||||
# Packet Station production settings on the production VM (native, no Docker).
|
||||
# deploy/vm/install.sh --stack prod copies this to /etc/packetstation/packetstation.env
|
||||
# (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
|
||||
# It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
|
||||
# Starting packetstation-worker with this file polls VISUAL production on its schedule.
|
||||
# Spencer's first start of the worker is the approval.
|
||||
#
|
||||
# systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
|
||||
# Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
|
||||
# Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
|
||||
|
||||
# ---- Stack and site ----
|
||||
# prod, test, ci or dev. Guards against crossing stacks.
|
||||
STACK=prod
|
||||
# The VISUAL site in every call. The docker-1 test stack runs with PESCO.
|
||||
SITE_ID=PESCO
|
||||
# Site time zone for schedules and display. The processes run in UTC.
|
||||
SITE_TZ=America/Denver
|
||||
# Public URL, used in links and mail.
|
||||
PUBLIC_URL=https://packetstation.pescoinc.biz
|
||||
# NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
|
||||
# opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
|
||||
TRUSTED_PROXY_HOST=192.168.101.12
|
||||
# DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
|
||||
# A folder means the Unix socket. PostgreSQL listens on no TCP port.
|
||||
PG_HOST=/var/run/postgresql
|
||||
PG_PORT=5432
|
||||
PG_DATABASE=packetstation
|
||||
PG_APP_USER=ps_app
|
||||
PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
|
||||
PG_MIGRATE_USER=ps_migrate
|
||||
PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
|
||||
|
||||
# ---- Keys and folders ----
|
||||
# The services get the secrets as systemd credentials and ps-launch.sh points these at them.
|
||||
# The paths here are the files those credentials are loaded from.
|
||||
SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
|
||||
SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
|
||||
# Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
|
||||
ASSETS_DIR=/var/lib/packetstation/assets
|
||||
# Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
|
||||
BACKUP_DIR=/var/backups/packetstation
|
||||
# The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
|
||||
# `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
|
||||
# Admin on the Admin page, File locations.
|
||||
#ARCHIVE_ROOT=/mnt/packetstation-archive
|
||||
|
||||
# ---- Uploads and storage ----
|
||||
UPLOAD_MAX_BYTES=26214400
|
||||
UPLOAD_MAX_BYTES_BOM=5242880
|
||||
ASSETS_QUOTA_WARN_PCT=85
|
||||
# 0 measures the file system under ASSETS_DIR.
|
||||
ASSETS_QUOTA_BYTES=0
|
||||
|
||||
# ---- ClamAV (optional, off: Gitea issue 19) ----
|
||||
# Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
|
||||
#CLAMAV_HOST=127.0.0.1
|
||||
CLAMAV_PORT=3310
|
||||
CLAMAV_TIMEOUT_SECONDS=30
|
||||
|
||||
# ---- LDAP (PESCO Active Directory) ----
|
||||
LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
|
||||
LDAP_BASE_DN=DC=PESCO,DC=com
|
||||
LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
|
||||
LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
|
||||
LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
|
||||
LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
|
||||
# The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
|
||||
LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
|
||||
LDAP_STARTTLS=false
|
||||
LDAP_ALLOW_INSECURE=false
|
||||
|
||||
# ---- Mail (relay app-03, no auth) ----
|
||||
SMTP_HOST=app-03.pesco.com
|
||||
SMTP_PORT=25
|
||||
SMTP_FROM=[email protected]
|
||||
SMTP_STARTTLS=true
|
||||
SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
|
||||
# Production sends mail.
|
||||
NOTIFY_DRY_RUN=false
|
||||
|
||||
# ---- VisualKit (VISUAL ERP, read only) ----
|
||||
# Production, set explicitly. Never on the test VM.
|
||||
VISUAL_SERVER=vsql.pesco.com
|
||||
VISUAL_DATABASE=PESCOINC
|
||||
VISUAL_USER=visualkit_ro
|
||||
VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
|
||||
# no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
|
||||
# store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
|
||||
VISUAL_ENCRYPT=no
|
||||
VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
|
||||
|
||||
# ---- MFilesKit (M-Files, read only) ----
|
||||
MFILES_URL=https://mfiles.pescoinc.biz/REST
|
||||
MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
|
||||
MFILES_USERNAME=mfileskitRO
|
||||
# ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
|
||||
MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
|
||||
# Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# `packetstation doctor` on this VM, run the way the services run (see ps-run.sh).
|
||||
# Installed as /usr/local/sbin/ps-doctor. Run as root: sudo ps-doctor [--skip visual,mfiles,ldap,smtp]
|
||||
# On production the VISUAL check connects to VISUAL production, read-only, as visualkit_ro.
|
||||
set -euo pipefail
|
||||
PS_RUN="${PS_RUN:-/usr/local/sbin/ps-run}"
|
||||
exec "$PS_RUN" doctor "$@"
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# Packet Station launcher on a VM. The native counterpart of docker/entrypoint.sh.
|
||||
# The systemd units and ps-run start every packetstation command through it.
|
||||
#
|
||||
# Usage: ps-launch.sh app | worker | migrate | doctor [...] | job run <name> | version | healthcheck app|worker | ...
|
||||
#
|
||||
# It runs from a release folder, /opt/packetstation/<release>/deploy/vm/ps-launch.sh, and uses that
|
||||
# release's venv and built frontend. systemd has already loaded /etc/packetstation/packetstation.env.
|
||||
#
|
||||
# Secrets: each unit hands its own secrets to the service as systemd credentials (LoadCredential=),
|
||||
# so the app never sees the VISUAL or M-Files password. When $CREDENTIALS_DIRECTORY holds a known
|
||||
# secret, the matching *_FILE setting is pointed at it. MFilesKit reads its password from the
|
||||
# environment only, so MFILES_PASSWORD is exported from MFILES_PASSWORD_FILE, as the entrypoint does.
|
||||
# No value is ever printed.
|
||||
#
|
||||
# Set here, over the env file: STATIC_DIR and PS_BUILD (from the release), WORKER_BEAT_FILE, TZ, LANG.
|
||||
set -euo pipefail
|
||||
|
||||
self="$(readlink -f -- "${BASH_SOURCE[0]}")"
|
||||
release="$(cd -- "$(dirname -- "$self")/../.." && pwd -P)"
|
||||
venv="${PS_VENV:-$release/venv}"
|
||||
|
||||
# secret name (the credential id and the file name in /etc/packetstation/secrets) and its setting
|
||||
cred_map='postgres_app_password PG_APP_PASSWORD_FILE
|
||||
postgres_migrate_password PG_MIGRATE_PASSWORD_FILE
|
||||
visualkit_password VISUAL_PASSWORD_FILE
|
||||
mfileskit_password MFILES_PASSWORD_FILE
|
||||
ldap_bind_password LDAP_PASSWORD_FILE
|
||||
session_key SESSION_KEY_FILE
|
||||
seal_key SEAL_KEY_FILE'
|
||||
|
||||
if [[ -n "${CREDENTIALS_DIRECTORY:-}" ]]; then
|
||||
while read -r name var; do
|
||||
if [[ -r "$CREDENTIALS_DIRECTORY/$name" ]]; then
|
||||
export "$var=$CREDENTIALS_DIRECTORY/$name"
|
||||
fi
|
||||
done <<< "$cred_map"
|
||||
fi
|
||||
unset cred_map name var
|
||||
|
||||
mfiles_secret="${MFILES_PASSWORD_FILE:-}"
|
||||
if [[ -n "$mfiles_secret" && -r "$mfiles_secret" ]]; then
|
||||
MFILES_PASSWORD="$(<"$mfiles_secret")"
|
||||
# Drop a trailing carriage return left by an editor on Windows.
|
||||
MFILES_PASSWORD="${MFILES_PASSWORD%$'\r'}"
|
||||
export MFILES_PASSWORD
|
||||
fi
|
||||
unset mfiles_secret
|
||||
|
||||
build="dev"
|
||||
if [[ -r "$release/BUILD" ]]; then
|
||||
build="$(head -n 1 -- "$release/BUILD")"
|
||||
build="${build%$'\r'}"
|
||||
fi
|
||||
|
||||
export PATH="$venv/bin:/usr/local/bin:/usr/bin:/bin"
|
||||
export STATIC_DIR="$release/static"
|
||||
export PS_BUILD="$build"
|
||||
# The worker unit makes /run/packetstation (RuntimeDirectory=). The heartbeat lives there so
|
||||
# `ps-run healthcheck worker` reads the same file the worker writes.
|
||||
export WORKER_BEAT_FILE=/run/packetstation/worker.beat
|
||||
# The same as the containers: UTC process clock, UTF-8, unbuffered logs to the journal.
|
||||
export TZ=UTC
|
||||
export LANG=C.UTF-8
|
||||
export PYTHONUNBUFFERED=1
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
unset build self
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
set -- app
|
||||
fi
|
||||
|
||||
exec "$venv/bin/packetstation" "$@"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run one packetstation command on this VM the way the services run it: as the packetstation user,
|
||||
# with the env file, the secrets as systemd credentials and the same sandbox, through systemd-run.
|
||||
# Output comes back to this terminal and the exit code is the command's.
|
||||
#
|
||||
# Installed as /usr/local/sbin/ps-run by deploy/vm/install.sh. Run it as root (sudo).
|
||||
#
|
||||
# Usage:
|
||||
# ps-run doctor [--skip visual,mfiles,ldap,smtp]
|
||||
# ps-run job run visual-feed
|
||||
# ps-run ad-groups --list
|
||||
# ps-run healthcheck worker
|
||||
# ps-run version
|
||||
#
|
||||
# Every secret file present in /etc/packetstation/secrets is handed over as a credential; the
|
||||
# folder itself stays out of reach of the command. No value is read or printed here.
|
||||
set -euo pipefail
|
||||
|
||||
ENV_FILE="${PS_ENV_FILE_PATH:-/etc/packetstation/packetstation.env}"
|
||||
SECRETS_DIR="${PS_SECRETS_DIR:-/etc/packetstation/secrets}"
|
||||
CURRENT="${PS_CURRENT:-/opt/packetstation/current}"
|
||||
SYSTEMD_RUN="${SYSTEMD_RUN:-systemd-run}"
|
||||
SERVICE_USER="${PS_USER:-packetstation}"
|
||||
ASSETS_DIR_RW="${PS_ASSETS_DIR:-/var/lib/packetstation/assets}"
|
||||
ARCHIVE_DIR_RW="${PS_ARCHIVE_MOUNT:-/mnt/packetstation-archive}"
|
||||
|
||||
die() { printf 'ps-run: %s\n' "$*" >&2; exit 2; }
|
||||
|
||||
if [[ $# -eq 0 || "$1" == "-h" || "$1" == "--help" ]]; then
|
||||
sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "${PS_RUN_ALLOW_NONROOT:-}" != "1" && "$(id -u)" != "0" ]]; then
|
||||
die "run as root (sudo ps-run ...)"
|
||||
fi
|
||||
[[ -r "$ENV_FILE" ]] || die "env file $ENV_FILE is missing"
|
||||
[[ -x "$CURRENT/deploy/vm/ps-launch.sh" ]] || die "no release at $CURRENT (run ps-upgrade first)"
|
||||
|
||||
props=(
|
||||
-p "User=$SERVICE_USER"
|
||||
-p "Group=$SERVICE_USER"
|
||||
-p "EnvironmentFile=$ENV_FILE"
|
||||
-p UMask=0027
|
||||
-p NoNewPrivileges=yes
|
||||
-p ProtectSystem=strict
|
||||
-p ProtectHome=yes
|
||||
-p PrivateTmp=yes
|
||||
-p PrivateDevices=yes
|
||||
-p ProtectKernelTunables=yes
|
||||
-p ProtectKernelModules=yes
|
||||
-p ProtectControlGroups=yes
|
||||
-p RestrictSUIDSGID=yes
|
||||
-p "ReadWritePaths=$ASSETS_DIR_RW"
|
||||
-p "ReadWritePaths=-$ARCHIVE_DIR_RW"
|
||||
-p "InaccessiblePaths=-$SECRETS_DIR"
|
||||
)
|
||||
for name in postgres_app_password postgres_migrate_password visualkit_password mfileskit_password \
|
||||
ldap_bind_password session_key seal_key; do
|
||||
if [[ -f "$SECRETS_DIR/$name" ]]; then
|
||||
props+=(-p "LoadCredential=$name:$SECRETS_DIR/$name")
|
||||
fi
|
||||
done
|
||||
|
||||
exec "$SYSTEMD_RUN" --wait --pipe --collect --quiet --service-type=exec \
|
||||
--description="Packet Station: packetstation $1" \
|
||||
"${props[@]}" -- "$CURRENT/deploy/vm/ps-launch.sh" "$@"
|
||||
@@ -0,0 +1,70 @@
|
||||
# Packet Station: the web app (uvicorn on port 8000, the API and the built frontend).
|
||||
# Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
|
||||
# NPM on docker-1 is the only client allowed on port 8000 (packetstation-firewall.service).
|
||||
[Unit]
|
||||
Description=Packet Station web app
|
||||
Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
|
||||
Requires=packetstation-migrate.service
|
||||
# Wants, not Requires: a restart of nftables.service must not restart the app. ExecStartPre checks the table.
|
||||
Wants=packetstation-firewall.service
|
||||
After=packetstation-migrate.service packetstation-firewall.service [email protected] network-online.target
|
||||
Wants=[email protected] network-online.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=packetstation
|
||||
Group=packetstation
|
||||
EnvironmentFile=/etc/packetstation/packetstation.env
|
||||
# The app's secrets only. It never gets the VISUAL or M-Files password.
|
||||
LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
|
||||
LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
|
||||
LoadCredential=session_key:/etc/packetstation/secrets/session_key
|
||||
LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
|
||||
# The app never listens without the firewall table in place ("+": this check runs as root).
|
||||
ExecStartPre=+/usr/sbin/nft list table inet packetstation
|
||||
ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh app
|
||||
SyslogIdentifier=packetstation-app
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
TimeoutStopSec=30s
|
||||
WorkingDirectory=/
|
||||
UMask=0027
|
||||
MemoryMax=1G
|
||||
LimitNOFILE=16384
|
||||
|
||||
# Sandbox. Read-only file system except the assets and the archive mount.
|
||||
# The leading "-" lets the app start while the archive is not mounted (ARCHIVE_ROOT unset).
|
||||
ReadWritePaths=/var/lib/packetstation/assets
|
||||
ReadWritePaths=-/mnt/packetstation-archive
|
||||
InaccessiblePaths=/etc/packetstation/secrets
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
# AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RemoveIPC=yes
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallErrorNumber=EPERM
|
||||
# Not set: MemoryDenyWriteExecute. pypdfium2 goes through ctypes, whose callbacks may need
|
||||
# writable and executable memory. Turn it on only after a filing and a print pass on the test VM.
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,21 @@
|
||||
# Packet Station: only NPM on docker-1 (TRUSTED_PROXY_HOST) and the VM itself may reach port 8000.
|
||||
# Loads /etc/packetstation/packetstation.nft, which deploy/vm/install.sh writes from the env file.
|
||||
# The table is Packet Station's own (inet packetstation); other rules on the VM are left alone.
|
||||
[Unit]
|
||||
Description=Packet Station firewall for the app port
|
||||
Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
|
||||
Before=network-pre.target packetstation-app.service
|
||||
Wants=network-pre.target
|
||||
# A restart of nftables.service flushes the whole ruleset. Load the table again after it.
|
||||
After=nftables.service
|
||||
PartOf=nftables.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
|
||||
ExecReload=/usr/sbin/nft -f /etc/packetstation/packetstation.nft
|
||||
ExecStop=-/usr/sbin/nft delete table inet packetstation
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,52 @@
|
||||
# Packet Station: SQL migrations, once per boot and once per upgrade, before the app and the worker.
|
||||
# Installed by deploy/vm/install.sh into /etc/systemd/system. See docs/DEPLOY.md, "Native VM install".
|
||||
# ps-upgrade restarts this unit after switching /opt/packetstation/current to a new release.
|
||||
[Unit]
|
||||
Description=Packet Station database migrations
|
||||
Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
|
||||
Wants=[email protected] network-online.target
|
||||
After=[email protected] network-online.target
|
||||
Before=packetstation-app.service packetstation-worker.service
|
||||
ConditionPathExists=/opt/packetstation/current/deploy/vm/ps-launch.sh
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
User=packetstation
|
||||
Group=packetstation
|
||||
EnvironmentFile=/etc/packetstation/packetstation.env
|
||||
# migrate needs both passwords: it logs in as ps_migrate and grants to ps_app.
|
||||
LoadCredential=postgres_migrate_password:/etc/packetstation/secrets/postgres_migrate_password
|
||||
LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
|
||||
ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh migrate
|
||||
SyslogIdentifier=packetstation-migrate
|
||||
TimeoutStartSec=15min
|
||||
WorkingDirectory=/
|
||||
UMask=0027
|
||||
|
||||
# Sandbox. The whole file system is read-only; migrate writes nothing outside PostgreSQL.
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
InaccessiblePaths=/etc/packetstation/secrets
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RemoveIPC=yes
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallErrorNumber=EPERM
|
||||
# No [Install]: the app and the worker pull this unit in (Requires=).
|
||||
@@ -0,0 +1,67 @@
|
||||
# Packet Station: the worker (VISUAL feed, drawings, AD reconcile, integrity, backup check,
|
||||
# notifications, archive copies). Installed by deploy/vm/install.sh into /etc/systemd/system.
|
||||
# On production its first start is Spencer's approval of the scheduled VISUAL polling.
|
||||
[Unit]
|
||||
Description=Packet Station worker
|
||||
Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
|
||||
Requires=packetstation-migrate.service
|
||||
After=packetstation-migrate.service [email protected] network-online.target
|
||||
Wants=[email protected] network-online.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=packetstation
|
||||
Group=packetstation
|
||||
EnvironmentFile=/etc/packetstation/packetstation.env
|
||||
LoadCredential=postgres_app_password:/etc/packetstation/secrets/postgres_app_password
|
||||
LoadCredential=visualkit_password:/etc/packetstation/secrets/visualkit_password
|
||||
LoadCredential=mfileskit_password:/etc/packetstation/secrets/mfileskit_password
|
||||
LoadCredential=ldap_bind_password:/etc/packetstation/secrets/ldap_bind_password
|
||||
LoadCredential=seal_key:/etc/packetstation/secrets/seal_key
|
||||
ExecStart=/opt/packetstation/current/deploy/vm/ps-launch.sh worker
|
||||
SyslogIdentifier=packetstation-worker
|
||||
# /run/packetstation holds the heartbeat file (WORKER_BEAT_FILE, set by ps-launch.sh).
|
||||
RuntimeDirectory=packetstation
|
||||
RuntimeDirectoryMode=0755
|
||||
Restart=on-failure
|
||||
RestartSec=10s
|
||||
TimeoutStopSec=60s
|
||||
WorkingDirectory=/
|
||||
UMask=0027
|
||||
MemoryMax=1G
|
||||
|
||||
# Sandbox. Read-only file system except the assets and the archive mount. The backup folder
|
||||
# (BACKUP_DIR) is only read, so it needs no entry here.
|
||||
ReadWritePaths=/var/lib/packetstation/assets
|
||||
ReadWritePaths=-/mnt/packetstation-archive
|
||||
InaccessiblePaths=/etc/packetstation/secrets
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
# AF_NETLINK: glibc's getaddrinfo asks the kernel for the local addresses through it.
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RemoveIPC=yes
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallErrorNumber=EPERM
|
||||
# Not set: MemoryDenyWriteExecute (see packetstation-app.service).
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,101 @@
|
||||
# Packet Station test settings on the test VM (native, no Docker).
|
||||
# deploy/vm/install.sh --stack test copies this to /etc/packetstation/packetstation.env
|
||||
# (root:packetstation 0640) if that file does not exist yet. It never overwrites it.
|
||||
# It holds no secret: passwords and keys are files in /etc/packetstation/secrets.
|
||||
# The test VM never points at VISUAL production. It reads VISUAL DEV.
|
||||
#
|
||||
# systemd reads this file (EnvironmentFile=). One KEY=value per line. Comments on their own lines.
|
||||
# Quote a value with spaces in double quotes. A value with a backslash goes in single quotes.
|
||||
# Not read from here: STATIC_DIR, PS_BUILD, WORKER_BEAT_FILE, TZ (ps-launch.sh sets them).
|
||||
|
||||
# ---- Stack and site ----
|
||||
# prod, test, ci or dev. Guards against crossing stacks.
|
||||
STACK=test
|
||||
# The VISUAL site in every call. The docker-1 test stack runs with PESCO.
|
||||
SITE_ID=PESCO
|
||||
# Site time zone for schedules and display. The processes run in UTC.
|
||||
SITE_TZ=America/Denver
|
||||
# Public URL, used in links and mail.
|
||||
PUBLIC_URL=https://packetstation-test.pescoinc.biz
|
||||
# NPM on docker-1, as the VM sees it. Only this address may set X-Forwarded-*. install.sh also
|
||||
# opens port 8000 to this address only (packetstation-firewall.service). Must be an IP address.
|
||||
TRUSTED_PROXY_HOST=192.168.101.12
|
||||
# DEBUG, INFO, WARNING or ERROR. Logs go to the journal.
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# ---- PostgreSQL 16 on this VM, over the local socket (roles made by install.sh) ----
|
||||
# A folder means the Unix socket. PostgreSQL listens on no TCP port.
|
||||
PG_HOST=/var/run/postgresql
|
||||
PG_PORT=5432
|
||||
PG_DATABASE=packetstation
|
||||
PG_APP_USER=ps_app
|
||||
PG_APP_PASSWORD_FILE=/etc/packetstation/secrets/postgres_app_password
|
||||
PG_MIGRATE_USER=ps_migrate
|
||||
PG_MIGRATE_PASSWORD_FILE=/etc/packetstation/secrets/postgres_migrate_password
|
||||
|
||||
# ---- Keys and folders ----
|
||||
# The services get the secrets as systemd credentials and ps-launch.sh points these at them.
|
||||
# The paths here are the files those credentials are loaded from.
|
||||
SESSION_KEY_FILE=/etc/packetstation/secrets/session_key
|
||||
SEAL_KEY_FILE=/etc/packetstation/secrets/seal_key
|
||||
# Stored files (drawings, MTRs, reports, data books). Owned by packetstation, 0750.
|
||||
ASSETS_DIR=/var/lib/packetstation/assets
|
||||
# Nightly dumps (ps-backup). The worker's backup-check job reads the markers here.
|
||||
BACKUP_DIR=/var/backups/packetstation
|
||||
# The archive NFS mount (Gitea issue 7). Uncomment once the share is mounted here and
|
||||
# `ps-run doctor` can write to it. Unset turns the archive off. The folder under it is set by an
|
||||
# Admin on the Admin page, File locations.
|
||||
#ARCHIVE_ROOT=/mnt/packetstation-archive
|
||||
|
||||
# ---- Uploads and storage ----
|
||||
UPLOAD_MAX_BYTES=26214400
|
||||
UPLOAD_MAX_BYTES_BOM=5242880
|
||||
ASSETS_QUOTA_WARN_PCT=85
|
||||
# 0 measures the file system under ASSETS_DIR.
|
||||
ASSETS_QUOTA_BYTES=0
|
||||
|
||||
# ---- ClamAV (optional, off: Gitea issue 19) ----
|
||||
# Empty means no scan. On a VM, clamd would run from the clamav-daemon package on 127.0.0.1.
|
||||
#CLAMAV_HOST=127.0.0.1
|
||||
CLAMAV_PORT=3310
|
||||
CLAMAV_TIMEOUT_SECONDS=30
|
||||
|
||||
# ---- LDAP (PESCO Active Directory) ----
|
||||
LDAP_URL="ldaps://DC-02.PESCO.COM:636 ldaps://PDC-1.PESCO.COM:636"
|
||||
LDAP_BASE_DN=DC=PESCO,DC=com
|
||||
LDAP_BIND_DN=CN=svc-packetstation,OU=System_Accounts,OU=Admin,DC=PESCO,DC=com
|
||||
LDAP_PASSWORD_FILE=/etc/packetstation/secrets/ldap_bind_password
|
||||
LDAP_USER_FILTER="(|(sAMAccountName={0})(userPrincipalName={0}))"
|
||||
LDAP_GROUP_BASE_DN=OU=PacketStation,OU=Groups,OU=PESCO_FARMINGTON,DC=PESCO,DC=com
|
||||
# The PESCO CA chain. install.sh --ca-bundle puts it here and in the system trust store.
|
||||
LDAP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
|
||||
LDAP_STARTTLS=false
|
||||
LDAP_ALLOW_INSECURE=false
|
||||
|
||||
# ---- Mail (relay app-03, no auth) ----
|
||||
SMTP_HOST=app-03.pesco.com
|
||||
SMTP_PORT=25
|
||||
SMTP_FROM=[email protected]
|
||||
SMTP_STARTTLS=true
|
||||
SMTP_CA_BUNDLE=/etc/packetstation/ca/pesco-ca.pem
|
||||
# The test VM never mails people about test data. The Admin "Send a test" still sends.
|
||||
NOTIFY_DRY_RUN=true
|
||||
|
||||
# ---- VisualKit (VISUAL ERP, read only) ----
|
||||
# VISUAL DEV. The test VM never points at production. Single quotes keep the backslash.
|
||||
VISUAL_SERVER='vsql.pesco.com\DEV'
|
||||
VISUAL_DATABASE=PESCOINCDEV
|
||||
VISUAL_USER=visualkit_ro
|
||||
VISUAL_PASSWORD_FILE=/etc/packetstation/secrets/visualkit_password
|
||||
# no until vsql has a trusted certificate (Gitea issue 5). The PESCO CA is in the VM's system
|
||||
# store (install.sh --ca-bundle), which the ODBC driver reads, so yes works once vsql has one.
|
||||
VISUAL_ENCRYPT=no
|
||||
VISUAL_DRIVER="ODBC Driver 18 for SQL Server"
|
||||
|
||||
# ---- MFilesKit (M-Files, read only) ----
|
||||
MFILES_URL=https://mfiles.pescoinc.biz/REST
|
||||
MFILES_VAULT_GUID={BB69270D-023E-4EA8-B899-AD20B1349702}
|
||||
MFILES_USERNAME=mfileskitRO
|
||||
# ps-launch.sh exports MFILES_PASSWORD from this file (the worker's credential).
|
||||
MFILES_PASSWORD_FILE=/etc/packetstation/secrets/mfileskit_password
|
||||
# Leave MFILES_CA_BUNDLE unset: mfiles.pescoinc.biz has a public Let's Encrypt certificate.
|
||||
Executable
+280
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install a Packet Station release on this VM and switch to it. Run by Spencer as root.
|
||||
# Installed as /usr/local/sbin/ps-upgrade by deploy/vm/install.sh.
|
||||
#
|
||||
# Usage:
|
||||
# ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]
|
||||
# ps-upgrade --rollback
|
||||
#
|
||||
# RELEASE.tar.gz comes from deploy/vm/make-release.sh. When RELEASE.tar.gz.sha256 lies next to it,
|
||||
# the archive is checked against it first.
|
||||
#
|
||||
# Steps:
|
||||
# 1. Unpack into /opt/packetstation/<build> and build its venv there: public pins from PyPI
|
||||
# (server/requirements.txt), visualkit and mfileskit from the Gitea index with --index-url
|
||||
# (server/requirements-pesco.txt), then the package itself with --no-deps; pip check.
|
||||
# A release folder that is already complete is reused.
|
||||
# 2. A backup (ps-backup) when the database exists, unless --no-backup.
|
||||
# 3. Stop the app and the worker, point /opt/packetstation/current at the new release
|
||||
# (previous keeps the old one), install its units and commands (install.sh --files-only).
|
||||
# 4. Run the migrations (restart packetstation-migrate.service). On failure: current goes back,
|
||||
# the services stay stopped, and the restore command is printed.
|
||||
# 5. Start again whichever of the app and the worker ran before. A first install starts nothing
|
||||
# unless --start: on production the first start is Spencer's approval of the VISUAL polling.
|
||||
# 6. Wait for /healthz, run ps-doctor (unless --no-doctor) and keep the newest 3 releases.
|
||||
#
|
||||
# --rollback points current back at previous and restarts what ran. Migrations are not undone:
|
||||
# roll back only to a release whose code works with the schema now in the database.
|
||||
#
|
||||
# Python: PYTHON (default python3.12) must be 3.12 or newer.
|
||||
set -euo pipefail
|
||||
umask 022
|
||||
|
||||
OPT="${PS_OPT:-/opt/packetstation}"
|
||||
PYTHON="${PYTHON:-python3.12}"
|
||||
PESCO_INDEX_URL="${PESCO_INDEX_URL:-https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/}"
|
||||
PYPI_INDEX_URL="${PYPI_INDEX_URL:-https://pypi.org/simple/}"
|
||||
KEEP_RELEASES="${KEEP_RELEASES:-3}"
|
||||
HEALTH_URL="http://127.0.0.1:8000/healthz"
|
||||
SERVICES=(packetstation-app.service packetstation-worker.service)
|
||||
|
||||
log() { printf 'ps-upgrade: %s\n' "$*" >&2; }
|
||||
die() { printf 'ps-upgrade: ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
usage() { sed -n '2,31p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
|
||||
tarball=""
|
||||
start=0
|
||||
backup=1
|
||||
doctor=1
|
||||
rollback=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--start) start=1; shift ;;
|
||||
--no-backup) backup=0; shift ;;
|
||||
--no-doctor) doctor=0; shift ;;
|
||||
--rollback) rollback=1; shift ;;
|
||||
-h|--help) usage ;;
|
||||
-*) log "unknown option $1"; usage ;;
|
||||
*) [[ -z "$tarball" ]] || usage; tarball="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$(id -u)" == "0" ]] || die "run as root"
|
||||
[[ $rollback -eq 1 || -n "$tarball" ]] || usage
|
||||
[[ "$KEEP_RELEASES" =~ ^[1-9][0-9]*$ ]] || die "KEEP_RELEASES must be a positive whole number"
|
||||
|
||||
exec 9>/run/lock/packetstation-upgrade.lock
|
||||
flock -n 9 || die "another ps-upgrade is running"
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
was_active=()
|
||||
remember_active() {
|
||||
local s
|
||||
was_active=()
|
||||
for s in "${SERVICES[@]}"; do
|
||||
if systemctl is-active --quiet "$s"; then was_active+=("$s"); fi
|
||||
done
|
||||
}
|
||||
|
||||
switch_to() {
|
||||
# Atomic: a new link next to current, then rename over it.
|
||||
local target="$1"
|
||||
ln -sfn -- "$target" "$OPT/.current.new"
|
||||
mv -T -- "$OPT/.current.new" "$OPT/current"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local i
|
||||
for (( i = 0; i < 30; i++ )); do
|
||||
if curl -fsS --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
|
||||
log "the app answers $HEALTH_URL"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
log "WARNING: the app did not answer $HEALTH_URL within 60 s: journalctl -u packetstation-app -n 50"
|
||||
return 1
|
||||
}
|
||||
|
||||
start_again() {
|
||||
local -a units=("$@")
|
||||
if [[ ${#units[@]} -eq 0 ]]; then
|
||||
log "the app and the worker were not running and stay stopped."
|
||||
log "start them with: systemctl enable --now packetstation-app packetstation-worker"
|
||||
return 0
|
||||
fi
|
||||
log "starting ${units[*]}"
|
||||
systemctl start "${units[@]}"
|
||||
local s
|
||||
for s in "${units[@]}"; do
|
||||
if [[ "$s" == packetstation-app.service ]]; then wait_healthy || true; fi
|
||||
done
|
||||
}
|
||||
|
||||
prune() {
|
||||
local cur prev d
|
||||
cur="$(readlink -f -- "$OPT/current" 2>/dev/null || true)"
|
||||
prev="$(readlink -f -- "$OPT/previous" 2>/dev/null || true)"
|
||||
local -a dirs=()
|
||||
while IFS= read -r d; do dirs+=("$d"); done < <(
|
||||
find "$OPT" -mindepth 1 -maxdepth 1 -type d -name '[A-Za-z0-9]*' -printf '%T@ %p\n' | sort -rn | cut -d' ' -f2-
|
||||
)
|
||||
# current and previous always stay; they count toward KEEP_RELEASES.
|
||||
local allowed=$(( KEEP_RELEASES - 2 )) kept=0
|
||||
for d in "${dirs[@]}"; do
|
||||
if [[ "$d" == "$cur" || "$d" == "$prev" ]]; then continue; fi
|
||||
kept=$(( kept + 1 ))
|
||||
if [[ $kept -gt $allowed ]]; then
|
||||
log "removing old release $(basename -- "$d")"
|
||||
rm -rf -- "$d"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ rollback
|
||||
|
||||
if [[ $rollback -eq 1 ]]; then
|
||||
[[ -L "$OPT/previous" ]] || die "no previous release to go back to"
|
||||
prev="$(readlink -f -- "$OPT/previous")"
|
||||
cur="$(readlink -f -- "$OPT/current")"
|
||||
[[ -f "$prev/.ready" ]] || die "previous release $prev is not complete"
|
||||
remember_active
|
||||
log "rolling back from $(basename -- "$cur") to $(basename -- "$prev")"
|
||||
if [[ ${#was_active[@]} -gt 0 ]]; then systemctl stop "${was_active[@]}"; fi
|
||||
switch_to "$prev"
|
||||
ln -sfn -- "$cur" "$OPT/previous"
|
||||
bash "$prev/deploy/vm/install.sh" --files-only
|
||||
start_again "${was_active[@]}"
|
||||
log "done. Migrations were not undone."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 1. unpack and build
|
||||
|
||||
[[ -r "$tarball" ]] || die "cannot read $tarball"
|
||||
if [[ -r "$tarball.sha256" ]]; then
|
||||
( cd -- "$(dirname -- "$tarball")" && sha256sum --quiet -c -- "$(basename -- "$tarball").sha256" ) \
|
||||
|| die "$tarball does not match $tarball.sha256"
|
||||
log "checksum ok"
|
||||
else
|
||||
log "no $tarball.sha256 next to the archive; checksum not checked"
|
||||
fi
|
||||
|
||||
# Every member must sit under one top folder packetstation-<build>/, with no absolute path and no "..".
|
||||
# The listing is read whole first: an early-exiting reader (head, grep -q) would make tar fail
|
||||
# with SIGPIPE under pipefail.
|
||||
members="$(tar -tzf "$tarball")" || die "cannot list $tarball"
|
||||
top="${members%%$'\n'*}"
|
||||
top="${top%%/*}"
|
||||
[[ "$top" =~ ^packetstation-([A-Za-z0-9][A-Za-z0-9._-]{0,63})$ ]] || die "unexpected top folder '$top' in $tarball"
|
||||
build="${BASH_REMATCH[1]}"
|
||||
outside="$(grep -Evc "^$top(/|$)" <<< "$members" || true)"
|
||||
[[ "$outside" == "0" ]] || die "$tarball has files outside $top/"
|
||||
dotdot="$(grep -Ec '(^|/)\.\.(/|$)' <<< "$members" || true)"
|
||||
[[ "$dotdot" == "0" ]] || die "$tarball has a '..' path"
|
||||
unset members
|
||||
|
||||
dest="$OPT/$build"
|
||||
if [[ -f "$dest/.ready" ]]; then
|
||||
log "release $build is already built in $dest"
|
||||
else
|
||||
"$PYTHON" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)' \
|
||||
|| die "$PYTHON is not Python 3.12 or newer (set PYTHON=/path/to/python3.12)"
|
||||
rm -rf -- "$dest"
|
||||
install -d -o root -g root -m 0755 "$dest"
|
||||
log "unpacking $build into $dest"
|
||||
tar -xzf "$tarball" -C "$dest" --strip-components=1 --no-same-owner --no-same-permissions
|
||||
[[ "$(head -n 1 "$dest/BUILD")" == "$build" ]] || die "BUILD in the archive does not say $build"
|
||||
[[ -f "$dest/static/index.html" ]] || die "the archive has no built frontend (static/index.html)"
|
||||
|
||||
log "building the venv"
|
||||
"$PYTHON" -m venv "$dest/venv"
|
||||
pip=("$dest/venv/bin/python" -m pip --disable-pip-version-check --no-input)
|
||||
# No pip.conf or PIP_* setting from the host may add an index.
|
||||
export PIP_CONFIG_FILE=/dev/null
|
||||
unset PIP_INDEX_URL PIP_EXTRA_INDEX_URL PIP_FIND_LINKS
|
||||
"${pip[@]}" install --index-url "$PYPI_INDEX_URL" -r "$dest/server/requirements.txt"
|
||||
# PESCO libraries from the Gitea index only. --index-url, never --extra-index-url.
|
||||
"${pip[@]}" install --index-url "$PESCO_INDEX_URL" -r "$dest/server/requirements-pesco.txt"
|
||||
# The package itself. The build backend (hatchling) comes from PyPI.
|
||||
"${pip[@]}" install --index-url "$PYPI_INDEX_URL" --no-deps "$dest"
|
||||
"${pip[@]}" check
|
||||
# The release is read-only to the services. Owned by root, nothing writable by others.
|
||||
chown -R root:root "$dest"
|
||||
chmod -R u+rwX,go+rX,go-w "$dest"
|
||||
# The units and ps-run start these directly; an archive made on Windows may have lost the x bit.
|
||||
chmod 0755 "$dest"/deploy/vm/*.sh "$dest"/scripts/backup/*.sh
|
||||
printf '%s\n' "$build" > "$dest/.ready"
|
||||
log "release $build built"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 2. backup
|
||||
|
||||
first_install=1
|
||||
[[ -L "$OPT/current" ]] && first_install=0
|
||||
|
||||
db_exists=0
|
||||
found="$(setpriv --reuid=postgres --regid=postgres --init-groups -- \
|
||||
psql -X -At -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'packetstation'" 2>/dev/null || true)"
|
||||
if [[ "$found" == "1" ]]; then
|
||||
db_exists=1
|
||||
fi
|
||||
if [[ $backup -eq 1 && $db_exists -eq 1 && $first_install -eq 0 ]]; then
|
||||
log "backup before the upgrade"
|
||||
/usr/local/sbin/ps-backup || die "the backup failed; nothing was changed. Fix it, or use --no-backup."
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 3. switch
|
||||
|
||||
remember_active
|
||||
if [[ $start -eq 1 ]]; then was_active=("${SERVICES[@]}"); fi
|
||||
old=""
|
||||
if [[ $first_install -eq 0 ]]; then
|
||||
old="$(readlink -f -- "$OPT/current")"
|
||||
fi
|
||||
if [[ "$old" == "$dest" ]]; then
|
||||
log "release $build is already current"
|
||||
fi
|
||||
|
||||
active_now=()
|
||||
for s in "${SERVICES[@]}"; do
|
||||
if systemctl is-active --quiet "$s"; then active_now+=("$s"); fi
|
||||
done
|
||||
if [[ ${#active_now[@]} -gt 0 ]]; then
|
||||
log "stopping ${active_now[*]}"
|
||||
systemctl stop "${active_now[@]}"
|
||||
fi
|
||||
|
||||
switch_to "$dest"
|
||||
if [[ -n "$old" && "$old" != "$dest" ]]; then
|
||||
ln -sfn -- "$old" "$OPT/previous"
|
||||
fi
|
||||
log "current is $build"
|
||||
bash "$dest/deploy/vm/install.sh" --files-only
|
||||
|
||||
# ------------------------------------------------------------------ 4. migrate
|
||||
|
||||
log "running the migrations"
|
||||
if ! systemctl restart packetstation-migrate.service; then
|
||||
journalctl -u packetstation-migrate.service -n 30 --no-pager >&2 || true
|
||||
if [[ -n "$old" && "$old" != "$dest" ]]; then
|
||||
switch_to "$old"
|
||||
bash "$old/deploy/vm/install.sh" --files-only || true
|
||||
log "current is back on $(basename -- "$old")"
|
||||
fi
|
||||
last="$(find /var/backups/packetstation -maxdepth 1 -name 'ps-*.dump' -printf '%f\n' 2>/dev/null | sort | tail -n 1)"
|
||||
die "the migrations failed. The app and the worker are stopped. Read the log above. If the database
|
||||
must go back to the backup taken before this upgrade, restore it (docs/DEPLOY.md, \"Restore on a VM\"):
|
||||
ps-restore --dump /var/backups/packetstation/${last:-<dump>} --db ps_restored"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 5. and 6. start, check, prune
|
||||
|
||||
start_again "${was_active[@]}"
|
||||
if [[ $doctor -eq 1 ]]; then
|
||||
log "doctor"
|
||||
/usr/local/sbin/ps-doctor || log "WARNING: the doctor reported a failure (see above)"
|
||||
fi
|
||||
prune
|
||||
log "done: $build"
|
||||
+256
-80
@@ -1,6 +1,6 @@
|
||||
# Deploying Packet Station
|
||||
|
||||
How Packet Station runs on `docker-1`. Sections not built yet are marked "planned, phase N". The design is in [PLAN.md](PLAN.md) under "Deployment and CI".
|
||||
How Packet Station runs. Production and test move to their own native Linux VMs in phase 10 ("Native VM install"). Until the test VM is ready the test stack runs in Docker on `docker-1`, which keeps CI. Sections not built yet are marked "planned, phase N". The design is in [PLAN.md](PLAN.md) under "Deployment and CI".
|
||||
|
||||
## Host facts
|
||||
|
||||
@@ -59,7 +59,7 @@ The PESCO CA chain is mounted into the containers once and referenced by `LDAP_C
|
||||
|
||||
## Env files
|
||||
|
||||
One folder per stack. The test stack: `/home/spencerm/packetstation/test/`, like the other compose stacks on docker-1 (decided 2026-10-18, Gitea issue 38). It holds `stack.env`, `secrets/`, `ca/pesco-ca.pem` and `backups/`, and its `stack.env` sets `ENV_FILE`, `SECRETS_DIR`, `CA_BUNDLE_FILE` and `BACKUP_DIR` to those paths. The compose files run from the checkout in `/home/spencerm/packetstation/repo/deploy`. On 2026-10-27 Spencer decided that production and test each run natively on their own Linux VM, without Docker, and that docker-1 keeps CI only (Gitea issue 37). The test stack stays here until the test VM is ready. The production lines in this file describe the old compose plan and are rewritten for the VMs in phase 10. Each folder holds `stack.env` (settings only, no secrets) and `secrets/`. The env file sets `PROJECT_NAME`, `SECRETS_DIR` and `APP_ALIAS`, which the compose files require. `.env.example` in the repo lists every setting.
|
||||
One folder per stack. The test stack: `/home/spencerm/packetstation/test/`, like the other compose stacks on docker-1 (decided 2026-10-18, Gitea issue 38). It holds `stack.env`, `secrets/`, `ca/pesco-ca.pem` and `backups/`, and its `stack.env` sets `ENV_FILE`, `SECRETS_DIR`, `CA_BUNDLE_FILE` and `BACKUP_DIR` to those paths. The compose files run from the checkout in `/home/spencerm/packetstation/repo/deploy`. On 2026-10-27 Spencer decided that production and test each run natively on their own Linux VM, without Docker, and that docker-1 keeps CI only (Gitea issue 37). The test stack stays here until the test VM is ready. The VMs keep their settings in `/etc/packetstation/packetstation.env` and their secrets in `/etc/packetstation/secrets/`: see "Native VM install". The production compose lines in this file are the old plan, kept for reference. Each folder holds `stack.env` (settings only, no secrets) and `secrets/`. The env file sets `PROJECT_NAME`, `SECRETS_DIR` and `APP_ALIAS`, which the compose files require. `.env.example` in the repo lists every setting.
|
||||
|
||||
Key settings:
|
||||
|
||||
@@ -132,7 +132,7 @@ The Gitea PyPI index `https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/
|
||||
|
||||
## NPM proxy host
|
||||
|
||||
One proxy host per stack in Nginx Proxy Manager. To be recorded under `deploy/` as well.
|
||||
One proxy host per stack in Nginx Proxy Manager, recorded in `deploy/npm-proxy-host.md`. For the VMs see "NPM proxy host for the VMs": only the forward target and `TRUSTED_PROXY_HOST` change.
|
||||
|
||||
- Domain: `packetstation-test.pescoinc.biz` (test) or `packetstation.pescoinc.biz` (production).
|
||||
- Scheme `http`, forward host the stack's alias (`packetstation-test-app` or `packetstation-app`), port `8000`.
|
||||
@@ -241,7 +241,7 @@ Docker's default address pool on docker-1 holds fifteen /16 bridge networks (172
|
||||
|
||||
## Doctor
|
||||
|
||||
`packetstation doctor` proves a host and a stack are ready. Run it with `compose run`, not `compose exec`: the entrypoint exports MFILES_PASSWORD from its secret file, and `exec` skips the entrypoint, so the M-Files password check fails under `exec` even though the running worker has it.
|
||||
`packetstation doctor` proves a host and a stack are ready. On a VM run `sudo ps-doctor` ("Commands on a VM"). On docker-1 run it with `compose run`, not `compose exec`: the entrypoint exports MFILES_PASSWORD from its secret file, and `exec` skips the entrypoint, so the M-Files password check fails under `exec` even though the running worker has it.
|
||||
|
||||
```bash
|
||||
docker compose -p packetstation-test --env-file ~/packetstation/test/stack.env \
|
||||
@@ -340,113 +340,289 @@ The app sends test messages and the worker sends everything else, through PESCO'
|
||||
|
||||
The doctor's SMTP line checks STARTTLS with certificate verification, without sending.
|
||||
|
||||
## Backup (phase 9)
|
||||
## Native VM install (phase 10)
|
||||
|
||||
Host scripts in `scripts/backup/`, run by root from a systemd timer. They run outside the app, so a broken app still backs up. [scripts/backup/README.md](../scripts/backup/README.md) has the same steps with more detail.
|
||||
Production and test each run natively on their own Linux VM, without Docker (Spencer, 2026-10-27, Gitea issue 37). docker-1 keeps CI only. The test VM is built first from the same steps, so the install, the backups, the restore and the doctor are rehearsed before production. The docker-1 test stack keeps running until the test VM is ready (see "Retiring the docker-1 test stack").
|
||||
|
||||
| File | Installed as |
|
||||
The kit is in `deploy/vm/`. Spencer runs everything that needs root. Nothing in it starts the worker by itself: on production, Spencer's first start of the worker is the approval of the scheduled VISUAL polling.
|
||||
|
||||
### What IT provides (open on issue 37)
|
||||
|
||||
- One VM per stack. Ubuntu 24.04 LTS is recommended: it ships Python 3.12, which the server needs (`requires-python >= 3.12`, pins resolved for 3.12). Debian 12 ships Python 3.11, so on Debian 12 a Python 3.12 has to be provided separately and passed to `ps-upgrade` as `PYTHON=/path/to/python3.12`. `install.sh` supports both and refuses anything else. Size: 2 vCPU, 4 GB RAM and 40 GB disk is a starting point (the app and the worker are capped at 1 GB each; the test stack's assets were 125 MB after a month of testing).
|
||||
- A fixed LAN address per VM. NPM forwards to it.
|
||||
- Outbound from the VM: the OS mirrors, `apt.postgresql.org`, `packages.microsoft.com`, `www.postgresql.org` (the PGDG key), `pypi.org` and `files.pythonhosted.org`, `gt.pescoinc.biz` (the Python index), and at run time vsql (1433, plus UDP 1434 for the named DEV instance), `mfiles.pescoinc.biz` 443, DC-02 and PDC-1 636, `app-03.pesco.com` 25, the QNAP (NFS 2049).
|
||||
- Inbound: SSH for Spencer, and port 8000 from docker-1 (192.168.101.12) only. The VM enforces the second itself.
|
||||
- PBS backs the VM up like any other, at 03:00.
|
||||
|
||||
### Layout on a VM
|
||||
|
||||
| Path | Owner, mode | What |
|
||||
|---|---|---|
|
||||
| `/opt/packetstation/<build>/` | root, 0755 | One release: the source, `BUILD`, `static/` (the built frontend) and `venv/`. Read-only to the services. |
|
||||
| `/opt/packetstation/current`, `previous` | root | Symlinks to the running and the last release. The units start `current/deploy/vm/ps-launch.sh`. |
|
||||
| `/etc/packetstation/packetstation.env` | root:packetstation 0640 | Settings, no secrets. From `deploy/vm/prod.env.example` or `test.env.example`. |
|
||||
| `/etc/packetstation/secrets/` | root:packetstation 0750, files 0440 | One file per secret (table below). The services get them as systemd credentials and cannot open the folder. |
|
||||
| `/etc/packetstation/ca/pesco-ca.pem` | root, 0644 | The PESCO CA chain for `LDAP_CA_BUNDLE` and `SMTP_CA_BUNDLE`. Also in the system trust store, which the ODBC driver uses. |
|
||||
| `/etc/packetstation/backup.conf`, `packetstation.nft` | root, 0644 | The backup stack line; the firewall table (written by `install.sh`). |
|
||||
| `/var/lib/packetstation/assets/` | packetstation, 0750 | `ASSETS_DIR`, the stored files. |
|
||||
| `/var/backups/packetstation/` | root:packetstation 0750 | `BACKUP_DIR`, the nightly dumps and markers. |
|
||||
| `/mnt/packetstation-archive/` | root, 0755 until mounted | `ARCHIVE_ROOT`, the QNAP NFS share. |
|
||||
| `/usr/local/sbin/ps-run`, `ps-doctor`, `ps-upgrade`, `ps-backup`, `ps-restore` | root, 0755 | The commands below. |
|
||||
|
||||
The service user is `packetstation`, uid and gid 10001: the image's uid, so the NFS share's rights and the old volume's file owners carry over. PostgreSQL 16 from the PGDG repository listens on the local socket only (`listen_addresses = ''`, server time zone UTC, `conf.d/packetstation.conf`). `pg_hba.conf` gets one block, before Debian's peer line, that lets `ps_app` and `ps_migrate` log in to `packetstation` on the socket with their passwords (scram-sha-256). `install.sh` creates the two roles and the database as `deploy/postgres/initdb` does (owner `ps_migrate`, UTF8, locale `en_US.UTF-8`, `template0`, CONNECT and TEMPORARY for `ps_app`, schema `public` owned by `ps_migrate`). The env file sets `PG_HOST=/var/run/postgresql`.
|
||||
|
||||
| Secret file | Made by | Given to |
|
||||
|---|---|---|
|
||||
| `postgres_migrate_password` | `install.sh --generate-secrets` | migrate |
|
||||
| `postgres_app_password` | `install.sh --generate-secrets` | migrate, app, worker |
|
||||
| `session_key` | `install.sh --generate-secrets` | app |
|
||||
| `seal_key` | `install.sh --generate-secrets` | app, worker |
|
||||
| `visualkit_password` | Spencer | worker |
|
||||
| `mfileskit_password` | Spencer | worker (`ps-launch.sh` exports `MFILES_PASSWORD`) |
|
||||
| `ldap_bind_password` | Spencer | app, worker |
|
||||
|
||||
There is no PostgreSQL superuser password on a VM: root reaches the `postgres` OS user, which PostgreSQL trusts on the socket. `ps-run` hands every secret file present to the command it runs.
|
||||
|
||||
### The units
|
||||
|
||||
| Unit | What |
|
||||
|---|---|
|
||||
| `ps-backup.sh` | `/usr/local/sbin/ps-backup.sh` (root, 0755) |
|
||||
| `ps-restore.sh` | `/usr/local/sbin/ps-restore.sh` (root, 0755) |
|
||||
| `backup.conf.example` | `/etc/packetstation/backup.conf` (root, 0644) |
|
||||
| `packetstation-backup.service` | `/etc/systemd/system/` (root, 0644) |
|
||||
| `packetstation-backup.timer` | `/etc/systemd/system/` (root, 0644) |
|
||||
| `packetstation-migrate.service` | Oneshot, `RemainAfterExit`: `packetstation migrate` as `ps_migrate`. Pulled in by the app and the worker (`Requires=`), so neither starts on a failed migration. `ps-upgrade` restarts it. |
|
||||
| `packetstation-app.service` | `packetstation app`, uvicorn on port 8000. Refuses to start without the firewall table. `Restart=on-failure`, `MemoryMax=1G`. |
|
||||
| `packetstation-worker.service` | `packetstation worker`. Heartbeat in `/run/packetstation/worker.beat`. `Restart=on-failure`, `MemoryMax=1G`. |
|
||||
| `packetstation-firewall.service` | Loads `/etc/packetstation/packetstation.nft`: its own table `inet packetstation`, which lets only `TRUSTED_PROXY_HOST` (NPM) and the VM itself reach port 8000. Nothing else on the VM is filtered by it. |
|
||||
| `packetstation-backup.service`, `.timer` | `ps-backup` at 02:00 America/Denver. |
|
||||
|
||||
`backup.conf` lists the stacks, one per line: name, container (or `project:<compose project>`), database, folder.
|
||||
The app, the worker and migrate run as `packetstation` with `EnvironmentFile=/etc/packetstation/packetstation.env` and a sandbox: `ProtectSystem=strict` (writable only `/var/lib/packetstation/assets` and `/mnt/packetstation-archive`, the latter optional so the app starts without the share), `ProtectHome`, `PrivateTmp`, `PrivateDevices`, `NoNewPrivileges`, no capabilities, `InaccessiblePaths=/etc/packetstation/secrets`, `SystemCallFilter=@system-service`, the address families `AF_UNIX AF_INET AF_INET6 AF_NETLINK`, and the kernel protections. `systemd-analyze security` rates the app 1.5 ("OK"). `MemoryDenyWriteExecute` is left off: pypdfium2 uses ctypes, whose callbacks may need writable executable memory. Try it on the test VM after a filing and a print pass before turning it on.
|
||||
|
||||
```
|
||||
test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
#prod project:packetstation packetstation /var/backups/packetstation/prod
|
||||
```
|
||||
Each unit loads only its own secrets with `LoadCredential=`, as compose did with secrets: the app never gets the VISUAL or M-Files password, and migrate gets only the two database passwords. `deploy/vm/ps-launch.sh` (the native counterpart of `docker/entrypoint.sh`) points each `*_FILE` setting at the credential, exports `MFILES_PASSWORD` from its file, and sets `STATIC_DIR` and `PS_BUILD` from the release, `WORKER_BEAT_FILE`, `TZ=UTC` and `LANG=C.UTF-8`. Those five are ignored in the env file.
|
||||
|
||||
The production line stays commented out until the production stack runs (phase 10). Until then it would fail every night.
|
||||
Logs go to the journal: `journalctl -u packetstation-app -u packetstation-worker -f`.
|
||||
|
||||
Where the dumps go:
|
||||
### Commands on a VM
|
||||
|
||||
- Test stack: `/home/spencerm/packetstation/test/backups`, owned by spencerm (the script gives new files the folder's owner and group). The test `stack.env` sets `BACKUP_DIR` to this path.
|
||||
- Production: `/var/backups/packetstation/prod`, `root:10001` 0750. The production `stack.env` sets `BACKUP_DIR=/var/backups/packetstation/prod`.
|
||||
| Command (as root) | What |
|
||||
|---|---|
|
||||
| `ps-doctor [--skip visual,mfiles,ldap,smtp,archive]` | `packetstation doctor`, run the way the services run. |
|
||||
| `ps-run <packetstation command>` | Any `packetstation` command as the service user, with the env file, the secrets as credentials and the sandbox, through `systemd-run`. For example `ps-run job run visual-feed`, `ps-run ad-groups --list`, `ps-run healthcheck worker`, `ps-run version`. |
|
||||
| `ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]` | Install a release and switch to it (below). |
|
||||
| `ps-upgrade --rollback` | Back to the previous release. |
|
||||
| `ps-backup`, `ps-restore` | "Backups on a VM", "Restore on a VM". |
|
||||
|
||||
The worker mounts the stack's folder read-only at `/backups`. Its `backup-check` job (07:15) only lists the folder, reads the dumps' sizes and times, and reads the markers.
|
||||
On production the doctor's VISUAL check connects to VISUAL production, read-only, as `visualkit_ro`. Use `--skip visual` before the go-live approval.
|
||||
|
||||
What a run does, per stack:
|
||||
### Release archives
|
||||
|
||||
1. Finds the stack's running postgres container (by compose labels for `project:`). None or two is a failure.
|
||||
2. `docker exec <pg> pg_dump -U postgres -d packetstation -Fc` over the local socket, which the postgres image trusts. No password is read or passed. Written under a temporary name, `.ps-<stamp>.partial`.
|
||||
3. `pg_dumpall --globals-only --no-role-passwords`: the roles without their password hashes. The role passwords are in the stack secrets.
|
||||
4. `pg_restore --list` with the container's own pg_restore must list at least one entry. Then the files are renamed to `ps-<UTC yyyymmddThhmmssZ>.dump` and `.globals.sql`, mode 0600.
|
||||
5. Keeps the newest 7 (`KEEP=7` in the unit, `--keep N`), with their globals files.
|
||||
6. Writes `last-success` or `last-failure` (`<ISO UTC time> <dump name>`, plus the reason on failure), mode 0644. A failed run leaves no partial file.
|
||||
A VM never builds the frontend and never runs Docker. `deploy/vm/make-release.sh` runs on a build host as a normal user, in a git checkout, and writes `packetstation-<build>.tar.gz` with a `.sha256`: the source of one commit (`pyproject.toml`, `README.md`, `server/`, `deploy/`, `scripts/`, `docs/`, with LF line endings), `BUILD`, and `static/`. Python packages are not in it.
|
||||
|
||||
Each stack runs on its own, so one failing does not stop the next. Exit 1 if any stack failed, 2 for a usage or config error. `flock` keeps two runs from overlapping.
|
||||
- `--image gt.pescoinc.biz/spencerm/packetstation:sha-<sha>` (on docker-1): copies `/app/static` out of the CI-tested image with `docker create` and `docker cp` (nothing runs) and takes the commit from the image's build label. The release is then the tested commit with the tested frontend. This is the normal way.
|
||||
- `--npm [--ref REF]` (on a PC with Node): builds the frontend with `npm ci` and `npm run build` from the same commit.
|
||||
|
||||
The unit is `Type=oneshot` as root, `Nice=10`, `IOSchedulingClass=idle`, `TimeoutStartSec=1h`, with `NoNewPrivileges`, `PrivateTmp` and `ProtectSystem=full` (no `ProtectHome`, so it can write the test folder under `/home`). The timer is `OnCalendar=*-*-* 02:00:00 America/Denver` with `Persistent=true`: 02:00 Denver time all year, an hour before the 03:00 PBS run, so PBS captures the fresh dumps. A missed run runs at the next boot.
|
||||
`ps-upgrade` builds the venv on the VM, as the image does: `server/requirements.txt` from PyPI with `--index-url https://pypi.org/simple/`, `server/requirements-pesco.txt` from the Gitea index with `--index-url` (never `--extra-index-url`), then the package with `--no-deps`, then `pip check`. Host pip settings are ignored (`PIP_CONFIG_FILE=/dev/null`).
|
||||
|
||||
### Install (Spencer, with sudo, on docker-1)
|
||||
### First install (Spencer, on a new VM)
|
||||
|
||||
After the phase 9 tree is synced to `~/packetstation/repo`:
|
||||
The test VM first. `<build>` is the release, for example `sha-<commit>`.
|
||||
|
||||
1. On docker-1, in a git checkout of the repository that has the commit (for example a clone of `gt.pescoinc.biz/spencerm/PacketStation`):
|
||||
|
||||
```bash
|
||||
deploy/vm/make-release.sh --image gt.pescoinc.biz/spencerm/packetstation:sha-<sha> --out ~/releases
|
||||
```
|
||||
|
||||
2. Copy `~/releases/packetstation-<build>.tar.gz`, its `.sha256` and the PESCO CA bundle to the VM. The bundle comes from `scripts/Export-PescoCaBundle.ps1 -NoCopy` (or `-Remote` with the VM's path), or from `~/packetstation/test/ca/pesco-ca.pem` on docker-1.
|
||||
|
||||
3. On the VM, unpack and run the install. It installs the packages, makes the user, folders, env file, CA, the four internal secrets, PostgreSQL's settings, roles and database, the units, the commands, the backup timer and the firewall:
|
||||
|
||||
```bash
|
||||
cd ~ && sha256sum -c packetstation-<build>.tar.gz.sha256
|
||||
tar -xzf packetstation-<build>.tar.gz
|
||||
sudo bash ./packetstation-<build>/deploy/vm/install.sh --stack test --ca-bundle ~/pesco-ca.pem --generate-secrets
|
||||
```
|
||||
|
||||
For production, `--stack prod`. If the VM is to carry on the docker-1 test stack's data, copy that stack's `seal_key` and `session_key` into `/etc/packetstation/secrets/` before this step instead of generating them: filed packets are sealed with the seal key.
|
||||
|
||||
4. Put the three outside secrets in place, without echo, then run the install again so it checks them and sets their owner and mode:
|
||||
|
||||
```bash
|
||||
for s in visualkit_password mfileskit_password ldap_bind_password; do
|
||||
sudo bash -c 'umask 077; read -rsp "$1: " v; echo; printf %s "$v" > "/etc/packetstation/secrets/$1"' _ "$s"
|
||||
done
|
||||
sudo bash ./packetstation-<build>/deploy/vm/install.sh --skip-packages
|
||||
```
|
||||
|
||||
5. Read `/etc/packetstation/packetstation.env` through (`sudoedit`). Check `STACK`, `PUBLIC_URL`, `VISUAL_SERVER` and `VISUAL_DATABASE` (test: `'vsql.pesco.com\DEV'` and `PESCOINCDEV`; production: `vsql.pesco.com` and `PESCOINC`), `NOTIFY_DRY_RUN` (test `true`), and `TRUSTED_PROXY_HOST=192.168.101.12`. After changing `TRUSTED_PROXY_HOST`, run `install.sh --files-only` to rewrite the firewall table.
|
||||
|
||||
6. Install the release. It builds the venv, runs the migrations and the doctor, and starts nothing on a first install:
|
||||
|
||||
```bash
|
||||
sudo ps-upgrade ~/packetstation-<build>.tar.gz
|
||||
```
|
||||
|
||||
7. Run the doctor until it is clean (on production with `--skip visual` until the approval):
|
||||
|
||||
```bash
|
||||
sudo ps-doctor
|
||||
```
|
||||
|
||||
8. The archive share, when the QNAP export exists ("Archive NFS mount").
|
||||
|
||||
9. Start. On production this is the approval of the scheduled VISUAL polling:
|
||||
|
||||
```bash
|
||||
sudo systemctl enable --now packetstation-app packetstation-worker
|
||||
systemctl status packetstation-app packetstation-worker --no-pager
|
||||
curl -fsS http://127.0.0.1:8000/healthz
|
||||
```
|
||||
|
||||
10. Point the NPM proxy host at the VM ("NPM proxy host for the VMs") and run its checks.
|
||||
|
||||
11. Backups: `sudo systemctl start packetstation-backup.service`, then `sudo ls -la /var/backups/packetstation`. Then rehearse a restore ("Restore on a VM").
|
||||
|
||||
On Ubuntu with `ufw` turned on, ufw also has to let docker-1 in: `sudo ufw allow from 192.168.101.12 to any port 8000 proto tcp`. The Packet Station table only narrows port 8000 further; it never opens anything ufw blocks.
|
||||
|
||||
### Upgrades
|
||||
|
||||
Build the archive for the new commit (step 1), copy it over (step 2), then:
|
||||
|
||||
```bash
|
||||
sudo ps-upgrade ~/packetstation-<build>.tar.gz
|
||||
```
|
||||
|
||||
In order: check the `.sha256` when it lies next to the archive, unpack into `/opt/packetstation/<build>` and build its venv (an already complete release folder is reused), take a backup with `ps-backup`, stop the app and the worker, point `current` at the new release (`previous` keeps the old), install its units and commands (`install.sh --files-only`), run the migrations, start again whichever of the two ran before, wait for `/healthz`, run the doctor, and keep the newest three releases. A short downtime window, as with a new image.
|
||||
|
||||
If the migrations fail, `current` goes back to the old release, the app and the worker stay stopped, and the restore command for the backup taken a minute earlier is printed. Read `journalctl -u packetstation-migrate`.
|
||||
|
||||
`sudo ps-upgrade --rollback` points `current` back at `previous` and restarts what ran. It does not undo migrations: roll back only to a release whose code works with the schema in the database, or restore the backup.
|
||||
|
||||
Open screens see the build change and offer a reload (the version banner).
|
||||
|
||||
### NPM proxy host for the VMs
|
||||
|
||||
NPM stays on docker-1 and keeps the certificates. The DNS names keep pointing at docker-1 (192.168.101.12). Only the forward target of each proxy host changes, to its VM over the LAN:
|
||||
|
||||
| | Production | Test |
|
||||
|---|---|---|
|
||||
| Domain name | `packetstation.pescoinc.biz` | `packetstation-test.pescoinc.biz` |
|
||||
| Scheme | `http` | `http` |
|
||||
| Forward hostname / IP | the production VM's address | the test VM's address |
|
||||
| Forward port | `8000` | `8000` |
|
||||
| `TRUSTED_PROXY_HOST` on the VM | `192.168.101.12` | `192.168.101.12` |
|
||||
|
||||
Everything else is as before (`deploy/npm-proxy-host.md`): Cache assets off, Block common exploits on, Websockets off, no access list; the PESCO certificate, Force SSL on, HTTP/2 on, NPM HSTS off (the app sends `Strict-Transport-Security` itself; two headers would conflict); and the same Advanced block, unchanged:
|
||||
|
||||
```nginx
|
||||
client_max_body_size 30m;
|
||||
|
||||
location /api/events {
|
||||
proxy_pass $forward_scheme://$server:$port;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_send_timeout 3600s;
|
||||
}
|
||||
```
|
||||
|
||||
With a fixed VM address the forward target can be the address itself; the "never an IP" rule was for container addresses, which change. `$server` keeps the events location on the same target as the rest of the host.
|
||||
|
||||
Trust: NPM's container reaches the VM through docker-1's own address, so the VM sees every proxied request from 192.168.101.12. `TRUSTED_PROXY_HOST=192.168.101.12` makes the app trust `X-Forwarded-For` and `X-Forwarded-Proto` from that address only (`proxy.py` resolves an address literal to itself; no server change was needed), and the firewall table lets only that address reach port 8000. Other hosts on the LAN cannot reach the app directly, and so cannot fake a client address for the sign-in rate limit. Traffic from NPM to the VM is plain HTTP on the LAN; TLS to the VM is still open on issue 37.
|
||||
|
||||
Check after switching: the four checks in `deploy/npm-proxy-host.md`, plus from another LAN PC `curl -m 5 http://<vm>:8000/healthz` must time out (the firewall), and the Admin page's access log shows the real client addresses, not 192.168.101.12.
|
||||
|
||||
### Archive NFS mount
|
||||
|
||||
The archive share (Gitea issue 7) lives on the QNAP. Spencer creates the NFS export with read and write for the VMs' addresses and makes the share's folder owned by uid 10001 and gid 10001 (or maps the VM's uid 10001 to an owner with write rights). Root squash on is fine: only `packetstation` writes there.
|
||||
|
||||
1. `nfs-common` is installed by `install.sh`. Add one line to `/etc/fstab` (the QNAP's name and export path from IT):
|
||||
|
||||
```
|
||||
<qnap>:/<export> /mnt/packetstation-archive nfs4 rw,soft,timeo=150,retrans=3,noatime,nosuid,nodev,noexec,_netdev,nofail 0 0
|
||||
```
|
||||
|
||||
`soft`: an unreachable QNAP makes a copy fail after about a minute instead of hanging the worker for good. The archive job writes each file to a temporary name, fsyncs and renames it, so a failed write shows up as a failed copy and is tried again with backoff. Filing never waits on it. `nofail` lets the VM boot without the QNAP.
|
||||
|
||||
2. Mount it and make the archive folder as the service user:
|
||||
|
||||
```bash
|
||||
sudo mount /mnt/packetstation-archive
|
||||
sudo setpriv --reuid=packetstation --regid=packetstation --init-groups mkdir /mnt/packetstation-archive/packets
|
||||
```
|
||||
|
||||
3. Set `ARCHIVE_ROOT=/mnt/packetstation-archive` in the env file (uncomment the line) and restart: `sudo systemctl restart packetstation-app packetstation-worker`.
|
||||
|
||||
4. On the Admin page, File locations, enter `packets`, press Check, then Save. `sudo ps-doctor` reports `archive` PASS from then on.
|
||||
|
||||
While the share is not mounted, the mount point is root's and read-only to `packetstation`, and `packets` does not exist under it, so nothing is written to the VM's own disk and the check fails with a plain message. The units list the mount point as an optional writable path, so the app and the worker start without it.
|
||||
|
||||
### Retiring the docker-1 test stack
|
||||
|
||||
The docker-1 test stack keeps running until the test VM passes its doctor, the NPM switch, a backup and a restore rehearsal. To carry its data over instead of starting empty: take a dump with the backup already installed on docker-1, tar the assets volume as in the 2026-10-21 rehearsal, copy both and the `seal_key` and `session_key` to the VM, and restore with `ps-restore` ("Restore on a VM"). Then on docker-1: stop the stack (`docker compose -p packetstation-test ... down`, keeping its volumes until the VM has run for a while), and remove its line from docker-1's `/etc/packetstation/backup.conf`. docker-1 keeps the old `ps-backup.sh` and `ps-restore.sh` it has installed; the versions in the repo are the VM ones and do not run there.
|
||||
|
||||
## Backups on a VM
|
||||
|
||||
`scripts/backup/ps-backup.sh` (installed as `/usr/local/sbin/ps-backup`) runs from `packetstation-backup.timer`, as root, outside the app, so a broken app still backs up. [scripts/backup/README.md](../scripts/backup/README.md) has the file list.
|
||||
|
||||
- Timer `OnCalendar=*-*-* 02:00:00 America/Denver`, `Persistent=true`: 02:00 Denver time all year, an hour before the 03:00 PBS run, so PBS captures the fresh dumps. A missed run runs at the next boot. `install.sh` enables it.
|
||||
- `/etc/packetstation/backup.conf`: one line, `name database folder`, for example `prod packetstation /var/backups/packetstation`. `install.sh` writes it.
|
||||
- `pg_dump -d packetstation -Fc` and `pg_dumpall --globals-only --no-role-passwords` run as the `postgres` OS user (`setpriv`, no PAM) on the local socket. No password is read or passed. Root writes the output under a temporary name, `pg_restore --list` must list at least one entry, then the files are renamed to `ps-<UTC yyyymmddThhmmssZ>.dump` and `.globals.sql`, mode 0600.
|
||||
- The newest 7 are kept (`KEEP=7` in the unit, `--keep N`), with their globals files. `last-success` or `last-failure` (`<ISO UTC time> <dump name>`, plus the reason on failure), mode 0644, owner and group taken from the folder (root:packetstation 0750). The worker's `backup-check` job (07:15) reads them through `BACKUP_DIR=/var/backups/packetstation`. A failed run leaves no partial file. `flock` keeps two runs apart. Exit 1 on a failure, 2 on a usage or config error.
|
||||
- The unit runs with `Nice=10`, `IOSchedulingClass=idle`, `TimeoutStartSec=1h`, `ProtectSystem=strict` (writable only `/var/backups/packetstation`), `ProtectHome`, `PrivateTmp`, `NoNewPrivileges`, and only the capabilities to switch user and set owners.
|
||||
- `ps-upgrade` takes a backup before every upgrade.
|
||||
|
||||
What the dump does not hold, and what covers it:
|
||||
|
||||
| What | Where | Covered by |
|
||||
|---|---|---|
|
||||
| Stored files | `/var/lib/packetstation/assets` | PBS (the VM). Restore with a PBS file-level restore and `ps-restore --assets`. |
|
||||
| Settings, secrets, CA | `/etc/packetstation` | PBS. The secrets are also needed to rebuild a VM: without `seal_key` the seals of filed packets cannot be checked. Keep a copy of `/etc/packetstation/secrets` where IT keeps the other service secrets. |
|
||||
| Archive copies | the QNAP share | The QNAP's own backup. |
|
||||
|
||||
Check by hand:
|
||||
|
||||
```bash
|
||||
cd ~/packetstation/repo
|
||||
sudo install -o root -g root -m 0755 scripts/backup/ps-backup.sh /usr/local/sbin/ps-backup.sh
|
||||
sudo install -o root -g root -m 0755 scripts/backup/ps-restore.sh /usr/local/sbin/ps-restore.sh
|
||||
sudo install -d -o root -g root -m 0755 /etc/packetstation
|
||||
sudo install -o root -g root -m 0644 scripts/backup/backup.conf.example /etc/packetstation/backup.conf
|
||||
sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.service /etc/systemd/system/
|
||||
sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemd-analyze verify /etc/systemd/system/packetstation-backup.service
|
||||
sudo systemctl enable --now packetstation-backup.timer
|
||||
systemctl list-timers packetstation-backup.timer
|
||||
sudo systemctl start packetstation-backup.service
|
||||
journalctl -u packetstation-backup.service -n 20 --no-pager
|
||||
ls -la ~/packetstation/test/backups
|
||||
sudo ls -la /var/backups/packetstation
|
||||
systemctl list-timers packetstation-backup.timer # next run 08:00 or 09:00 UTC, 02:00 in Denver
|
||||
```
|
||||
|
||||
`list-timers` shows the next run at 08:00 or 09:00 UTC (02:00 in Denver). The one run by hand should leave a new `ps-*.dump` and a fresh `last-success`. After changing a script in the repo, run its `install` line again: the installed copies are root's, so a sync of the checkout does not change what root runs.
|
||||
## Restore on a VM
|
||||
|
||||
When the production stack exists (phase 10):
|
||||
`ps-restore` (`scripts/backup/ps-restore.sh`) never stops, starts or reconfigures a service. It restores into a new database next to the live one, and a person decides how to switch. A whole-VM restore from PBS stays the disaster path.
|
||||
|
||||
```bash
|
||||
sudo install -d -o root -g root -m 0755 /var/backups/packetstation
|
||||
sudo install -d -o root -g 10001 -m 0750 /var/backups/packetstation/prod
|
||||
sudoedit /etc/packetstation/backup.conf # uncomment the prod line
|
||||
sudo systemctl start packetstation-backup.service
|
||||
```
|
||||
B=/var/backups/packetstation
|
||||
D=$(sudo ls "$B" | grep '\.dump$' | tail -1)
|
||||
|
||||
A run by hand needs no sudo for the test stack (spencerm is in the docker group):
|
||||
# 1. Into a fresh database. Refuses an existing one; --replace drops it first (DROP refuses while anyone is
|
||||
# connected). --replace is refused for packetstation itself.
|
||||
sudo ps-restore --dump "$B/$D" --globals "$B/${D%.dump}.globals.sql" --db ps_restored
|
||||
|
||||
```bash
|
||||
scripts/backup/ps-backup.sh --stack test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
```
|
||||
# 2. Stored files, from a folder or a tar (a PBS file-level restore). Existing files are never overwritten.
|
||||
# Files end up owned by packetstation. Use a scratch folder for a rehearsal.
|
||||
sudo ps-restore --assets /path/to/assets.tar --assets-dir /var/lib/packetstation/assets
|
||||
|
||||
The dumps hold the database only. The assets volume (the files) is not in them. It is covered by PBS, since the volume is part of the VM.
|
||||
|
||||
## Restore (phase 9)
|
||||
|
||||
`ps-restore.sh` never stops, restarts or reconfigures a stack. It restores into a new database next to the live one, and a person decides how to switch. A whole-VM restore from PBS stays the disaster path.
|
||||
|
||||
```bash
|
||||
B=/home/spencerm/packetstation/test/backups
|
||||
D=$(ls $B/ps-*.dump | tail -1)
|
||||
|
||||
# 1. Into a fresh database. Refuses an existing one; --replace drops it first (DROP refuses while anyone is connected).
|
||||
ps-restore.sh --project packetstation-test --dump "$D" --globals "${D%.dump}.globals.sql" --db ps_restored
|
||||
|
||||
# 2. Assets, from a folder or a tar (for example a PBS file-level restore of the assets volume folder).
|
||||
# Existing files are never overwritten. Files end up owned by 10001:10001.
|
||||
ps-restore.sh --project packetstation-test --assets /path/to/assets.tar --assets-volume packetstation-test_assets
|
||||
|
||||
# 3. Every asset row's sha256 against the files in the volume.
|
||||
ps-restore.sh --project packetstation-test --verify-assets --db ps_restored --assets-volume packetstation-test_assets
|
||||
# 3. Every asset row's sha256 against the files.
|
||||
sudo ps-restore --verify-assets --db ps_restored --assets-dir /var/lib/packetstation/assets
|
||||
|
||||
# Row counts of any database, to compare with the live one.
|
||||
ps-restore.sh --project packetstation-test --counts-only --db packetstation
|
||||
sudo ps-restore --counts-only --db packetstation
|
||||
```
|
||||
|
||||
- The database is created as `01-roles.sh` creates it (owner `ps_migrate`, UTF8, `template0`, CONNECT and TEMPORARY for `ps_app`), then `pg_restore --exit-on-error --single-transaction` as postgres. A failed restore leaves an empty database, never a half one.
|
||||
- The globals file is applied without stopping on "already exists". Any other error fails. The script refuses a globals file that holds a password hash. On a new cluster the roles come back without passwords: set each from the stack's secret file with `\password` in psql, or start the stack on an empty volume so initdb creates them.
|
||||
- The assets copy runs in a throwaway container from the image the postgres container already runs, with `--network none`, so nothing is downloaded.
|
||||
- To put a restored database into service: stop that stack's app and worker, rename as postgres (`ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`), start the stack, check the app, then drop the old database. This step is deliberate and manual.
|
||||
- The database is created as `install.sh` creates the live one (owner `ps_migrate`, UTF8, `en_US.UTF-8`, `template0`, CONNECT and TEMPORARY for `ps_app`), then `pg_restore --exit-on-error --single-transaction` as postgres. A failed restore leaves an empty database, never a half one.
|
||||
- The globals file is applied without stopping on "already exists". Any other error fails. A globals file with a password hash is refused. Roles created from it have no password: run `install.sh --skip-packages`, which sets both role passwords from the secrets.
|
||||
- To put a restored database into service: `sudo systemctl stop packetstation-app packetstation-worker`; as postgres (`sudo setpriv --reuid=postgres --regid=postgres --init-groups psql -X -d postgres`) run `ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`; then `sudo systemctl restart packetstation-migrate` and `sudo systemctl start packetstation-app packetstation-worker` (migrate brings an older dump up to the release), check the app, and drop the old database. This step is deliberate and manual.
|
||||
- A rebuilt VM after a disaster: copy the saved `/etc/packetstation/secrets` back first, run `install.sh`, `ps-upgrade` with the same release, then the restore and the rename above (the install made an empty `packetstation`), and the assets.
|
||||
- The next rehearsal is IT's, quarterly (requirement O2). The first one on the test VM is part of its acceptance.
|
||||
|
||||
## Backup and restore on docker-1 (phase 9, until the test VM)
|
||||
|
||||
Until the test VM takes over, the docker-1 test stack is backed up by the phase 9 scripts installed on docker-1 as `/usr/local/sbin/ps-backup.sh` and `ps-restore.sh`. They reach the stack's postgres container with `docker exec` and read `/etc/packetstation/backup.conf` in the four-column form `name container database folder` (`test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups`). Those versions are in git history (up to commit 3fdf063). The scripts in the repo are now the VM ones and refuse that config, so do not reinstall from the repo on docker-1. Markers, rotation and timer are the same as on a VM. The rehearsal below was made with them.
|
||||
|
||||
### Restore rehearsal 2026-10-21
|
||||
|
||||
|
||||
+1
-3
@@ -2,9 +2,7 @@
|
||||
|
||||
## backup/
|
||||
|
||||
The nightly per-stack `pg_dump` backup with its systemd timer, and the restore script. See
|
||||
[backup/README.md](backup/README.md) for what they write and Spencer's install steps. The restore
|
||||
rehearsal is recorded in docs/DEPLOY.md.
|
||||
The nightly `pg_dump` backup with its systemd timer, and the restore script, for the native VMs (no Docker). See [backup/README.md](backup/README.md) and docs/DEPLOY.md, "Backups on a VM" and "Restore on a VM". The VM install kit itself (units, launcher, install and upgrade) is in `deploy/vm/`.
|
||||
|
||||
## Export-PescoCaBundle.ps1
|
||||
|
||||
|
||||
+15
-85
@@ -1,99 +1,29 @@
|
||||
# Backup and restore
|
||||
|
||||
Host scripts for docker-1. They run outside the app, so a broken app still backs up.
|
||||
Scripts for the native VMs (production and test). They run outside the app, so a broken app still backs up. `deploy/vm/install.sh` installs them; the steps and the restore procedure are in [docs/DEPLOY.md](../../docs/DEPLOY.md), "Backups on a VM" and "Restore on a VM".
|
||||
|
||||
| File | What it is |
|
||||
|---|---|
|
||||
| `ps-backup.sh` | Nightly `pg_dump -Fc` of each stack's database plus `pg_dumpall --globals-only --no-role-passwords`, checked with `pg_restore --list`, 7 kept, `last-success` and `last-failure` markers. |
|
||||
| `ps-restore.sh` | Restores a dump into a fresh database, applies the globals, copies assets into a volume, checks asset hashes, prints row counts. |
|
||||
| `packetstation-backup.service` | systemd oneshot that runs `ps-backup.sh` as root. |
|
||||
| `packetstation-backup.timer` | 02:00 America/Denver every day, `Persistent=true`. |
|
||||
| `backup.conf.example` | The stack list for `/etc/packetstation/backup.conf`. |
|
||||
| File | Installed as | What it is |
|
||||
|---|---|---|
|
||||
| `ps-backup.sh` | `/usr/local/sbin/ps-backup` | Nightly `pg_dump -Fc` plus `pg_dumpall --globals-only --no-role-passwords`, as the postgres OS user on the local socket. Checked with `pg_restore --list`, 7 kept, `last-success` and `last-failure` markers. |
|
||||
| `ps-restore.sh` | `/usr/local/sbin/ps-restore` | Restores a dump into a fresh database, applies the globals, copies stored files into an assets folder without overwriting, checks every file's sha256 against the database, prints row counts. |
|
||||
| `packetstation-backup.service` | `/etc/systemd/system/` | Oneshot as root, sandboxed, writes only `/var/backups/packetstation`. |
|
||||
| `packetstation-backup.timer` | `/etc/systemd/system/` | 02:00 America/Denver every day, `Persistent=true`. |
|
||||
| `backup.conf.example` | `/etc/packetstation/backup.conf` | `name database folder`, one line per stack. |
|
||||
|
||||
Test: `tests/server/test_backup_scripts.py` runs `ps-backup.sh` with a fake `docker` on PATH.
|
||||
Tests: `tests/server/test_backup_scripts.py` runs both scripts with fake PostgreSQL tools (`PG_BINDIR`) and a fake user switch (`PG_AS`).
|
||||
|
||||
## What a backup is
|
||||
|
||||
In each stack's folder (`/var/backups/packetstation/prod`, `/home/spencerm/packetstation/test/backups`):
|
||||
In `/var/backups/packetstation` (root:packetstation 0750):
|
||||
|
||||
- `ps-<UTC stamp>.dump`: the app database, custom format. Mode 0600.
|
||||
- `ps-<UTC stamp>.globals.sql`: the roles, without passwords. Mode 0600.
|
||||
- `last-success`, `last-failure`: one line, `<ISO UTC time> <dump name>` (a failure adds the reason). Mode 0644. The worker's backup-check job compares their modification times.
|
||||
- `last-success`, `last-failure`: one line, `<ISO UTC time> <dump name>` (a failure adds the reason). Mode 0644. The worker's backup-check job reads them through the folder's group.
|
||||
|
||||
Each dump is written under a temporary name (`.ps-….partial`), checked, then renamed. A failed run leaves no partial file and writes `last-failure`. The script tries every stack and exits 1 if any failed, 2 for a usage or config error.
|
||||
Each dump is written under a temporary name (`.ps-….partial`), checked, then renamed. A failed run leaves no partial file and writes `last-failure`. Exit 1 if any stack failed, 2 for a usage or config error.
|
||||
|
||||
The script connects inside the postgres container as `postgres` over the local socket, which the image trusts. It reads no password.
|
||||
Not in the dump: the stored files (`/var/lib/packetstation/assets`), `/etc/packetstation` (env file, secrets, CA) and the archive share. The PBS backup of the VM covers the first two; the archive is on the QNAP.
|
||||
|
||||
Why `--no-role-passwords`: a password hash in a backup file is something to crack offline, and the file also lands in the PBS copy of the VM. The role passwords are in the stack's secrets already. On a fresh volume `deploy/postgres/initdb/01-roles.sh` creates the roles with them. On an existing cluster the roles are already there.
|
||||
## The docker-1 test stack
|
||||
|
||||
Ownership: run as root, the new files take the owner and group of the stack's folder, so the test folder stays spencerm's. The worker mounts the folder read-only as uid 10001. It only needs to list the folder and stat the dumps, plus read the markers, so dumps can stay 0600.
|
||||
|
||||
## Install (Spencer, with sudo, on docker-1)
|
||||
|
||||
Copy the files from the synced checkout in `~/packetstation/repo`. The installed copies are root's, so a later sync of the checkout does not change what root runs.
|
||||
|
||||
```bash
|
||||
cd ~/packetstation/repo
|
||||
sudo install -o root -g root -m 0755 scripts/backup/ps-backup.sh /usr/local/sbin/ps-backup.sh
|
||||
sudo install -o root -g root -m 0755 scripts/backup/ps-restore.sh /usr/local/sbin/ps-restore.sh
|
||||
sudo install -d -o root -g root -m 0755 /etc/packetstation
|
||||
sudo install -o root -g root -m 0644 scripts/backup/backup.conf.example /etc/packetstation/backup.conf
|
||||
sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.service /etc/systemd/system/
|
||||
sudo install -o root -g root -m 0644 scripts/backup/packetstation-backup.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemd-analyze verify /etc/systemd/system/packetstation-backup.service
|
||||
sudo systemctl enable --now packetstation-backup.timer
|
||||
systemctl list-timers packetstation-backup.timer # next run 08:00 or 09:00 UTC, 02:00 in Denver
|
||||
sudo systemctl start packetstation-backup.service # one run now
|
||||
journalctl -u packetstation-backup.service -n 20 --no-pager
|
||||
ls -la ~/packetstation/test/backups
|
||||
```
|
||||
|
||||
The config lists the test stack only. When the production stack exists (phase 10), make its folder and uncomment its line:
|
||||
|
||||
```bash
|
||||
sudo install -d -o root -g root -m 0755 /var/backups/packetstation
|
||||
sudo install -d -o root -g 10001 -m 0750 /var/backups/packetstation/prod
|
||||
sudoedit /etc/packetstation/backup.conf # uncomment the prod line
|
||||
sudo systemctl start packetstation-backup.service
|
||||
```
|
||||
|
||||
Group 10001 lets the production worker (uid 10001, group 10001) list the folder and read the markers. Its `stack.env` sets `BACKUP_DIR=/var/backups/packetstation/prod`.
|
||||
|
||||
After changing a script in the repo, install it again with the same `install` line.
|
||||
|
||||
## Run by hand
|
||||
|
||||
As spencerm (in the docker group), against the test stack:
|
||||
|
||||
```bash
|
||||
scripts/backup/ps-backup.sh --stack test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
```
|
||||
|
||||
`--stack NAME CONTAINER DATABASE FOLDER` can repeat. CONTAINER is a container name or `project:<compose project>`. `--keep N` (or `KEEP=N`) changes the 7.
|
||||
|
||||
## Restore
|
||||
|
||||
The script never stops, restarts or reconfigures a stack. Restore into a new database, check it, then decide how to switch.
|
||||
|
||||
```bash
|
||||
B=/home/spencerm/packetstation/test/backups
|
||||
D=$(ls $B/ps-*.dump | tail -1)
|
||||
|
||||
# 1. Into a fresh database (refuses an existing one; --replace drops it first, and DROP refuses while anyone is connected).
|
||||
ps-restore.sh --project packetstation-test --dump "$D" --globals "${D%.dump}.globals.sql" --db ps_restored
|
||||
|
||||
# 2. Assets: from a folder or a tar (for example a file-level restore out of PBS of the assets volume).
|
||||
# Existing files are never overwritten. Files end up owned by 10001:10001.
|
||||
ps-restore.sh --project packetstation-test --assets /path/to/assets.tar --assets-volume packetstation-test_assets
|
||||
|
||||
# 3. Check every asset row's sha256 against the files in the volume.
|
||||
ps-restore.sh --project packetstation-test --verify-assets --db ps_restored --assets-volume packetstation-test_assets
|
||||
|
||||
# Row counts of any database, for comparing.
|
||||
ps-restore.sh --project packetstation-test --counts-only --db packetstation
|
||||
```
|
||||
|
||||
To put the restored database into service, stop the app and worker of that stack, rename the databases as `postgres` (`ALTER DATABASE packetstation RENAME TO packetstation_old; ALTER DATABASE ps_restored RENAME TO packetstation;`), start the stack again, and drop the old one once the app is checked. That step is deliberate and manual.
|
||||
|
||||
The globals file holds no passwords. If a role had to be created from it on a new cluster, set its password from the stack's secret file with `\password` in psql, or start the stack on an empty volume so initdb creates the roles.
|
||||
These scripts no longer talk to Docker. The docker-1 test stack keeps the copies installed in `/usr/local/sbin/ps-backup.sh` and `ps-restore.sh` (the phase 9 versions, in git history up to commit 3fdf063) until it is stopped. Do not install the new scripts on docker-1: its `backup.conf` has the old four-column format, which the new script refuses.
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
# /etc/packetstation/backup.conf: the stacks ps-backup.sh backs up, one per line.
|
||||
# name container (or project:<compose project>) database folder
|
||||
test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
# Uncomment once the production stack runs (phase 10). Until then it would fail every night.
|
||||
#prod project:packetstation packetstation /var/backups/packetstation/prod
|
||||
# /etc/packetstation/backup.conf: the stacks ps-backup backs up, one per line.
|
||||
# deploy/vm/install.sh writes the VM's own line when the file is missing. A VM has one stack.
|
||||
# name database folder
|
||||
prod packetstation /var/backups/packetstation
|
||||
|
||||
@@ -1,25 +1,29 @@
|
||||
# Packet Station nightly database backup (scripts/backup/ps-backup.sh).
|
||||
# Installed by root on docker-1: see scripts/backup/README.md. Started by packetstation-backup.timer.
|
||||
# Runs as root because it talks to the Docker daemon and writes /var/backups/packetstation.
|
||||
# Packet Station nightly database backup on a native VM (scripts/backup/ps-backup.sh, installed as
|
||||
# /usr/local/sbin/ps-backup by deploy/vm/install.sh). Started by packetstation-backup.timer.
|
||||
# Runs as root: it reads the backup config, writes /var/backups/packetstation and switches to the
|
||||
# postgres OS user (setpriv) for pg_dump on the local socket.
|
||||
[Unit]
|
||||
Description=Packet Station database backup (pg_dump of each stack)
|
||||
Documentation=file:///home/spencerm/packetstation/repo/scripts/backup/README.md
|
||||
Wants=docker.service
|
||||
After=docker.service
|
||||
Description=Packet Station database backup (pg_dump)
|
||||
Documentation=file:///opt/packetstation/current/docs/DEPLOY.md
|
||||
Wants=[email protected]
|
||||
After=[email protected]
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/ps-backup.sh --config /etc/packetstation/backup.conf
|
||||
ExecStart=/usr/local/sbin/ps-backup --config /etc/packetstation/backup.conf
|
||||
Environment=KEEP=7
|
||||
# Gentle on the host: the dump runs while the shop is closed, but the stacks stay up.
|
||||
# Gentle on the VM: the dump runs while the shop is closed, but the app stays up.
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=1h
|
||||
# Hardening. /usr, /boot and /etc are read-only to the script; it writes only the backup folders.
|
||||
# Sandbox. Read-only file system except the backup folder. setpriv needs CAP_SETUID and CAP_SETGID.
|
||||
NoNewPrivileges=yes
|
||||
PrivateTmp=yes
|
||||
ProtectSystem=full
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ReadWritePaths=/var/backups/packetstation
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_CHOWN CAP_FOWNER CAP_DAC_OVERRIDE
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Packet Station nightly backup at 02:00 Denver time, an hour before the 03:00 Proxmox Backup
|
||||
# Server run, so PBS captures the fresh dumps. The host clock is UTC; the zone makes the intent
|
||||
# Server run, so PBS captures the fresh dumps. The VM clock may be UTC; the zone makes the intent
|
||||
# explicit and follows daylight saving. Persistent=true runs a missed backup at the next boot.
|
||||
[Unit]
|
||||
Description=Packet Station nightly database backup
|
||||
|
||||
+44
-60
@@ -1,50 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# Packet Station nightly backup. Runs on the docker host, never inside the app.
|
||||
# Packet Station nightly backup on a native VM. Runs as root from packetstation-backup.timer,
|
||||
# outside the app, so a broken app still backs up.
|
||||
#
|
||||
# For each configured stack it writes, into that stack's backup folder:
|
||||
# ps-<UTC stamp>.dump pg_dump -Fc of the app database (custom format)
|
||||
# ps-<UTC stamp>.globals.sql pg_dumpall --globals-only --no-role-passwords (roles, no password hashes)
|
||||
# last-success / last-failure one line: "<ISO UTC time> <dump name>" (last-failure adds the reason)
|
||||
#
|
||||
# Each dump is written under a temporary name, checked with `pg_restore --list` run inside the
|
||||
# postgres container, and only then renamed into place. The newest KEEP dumps (default 7) are kept,
|
||||
# with their globals files. Any failure writes last-failure and the script exits non-zero after
|
||||
# trying the remaining stacks.
|
||||
# Each dump is written under a temporary name, checked with `pg_restore --list`, and only then
|
||||
# renamed into place. The newest KEEP dumps (default 7) are kept, with their globals files. Any
|
||||
# failure writes last-failure and the script exits non-zero after trying the remaining stacks.
|
||||
#
|
||||
# Usage:
|
||||
# ps-backup.sh [--config FILE] [STACK ...]
|
||||
# ps-backup.sh --stack NAME CONTAINER DATABASE FOLDER [--stack ...]
|
||||
# ps-backup [--config FILE] [--keep N] [STACK ...]
|
||||
# ps-backup --stack NAME DATABASE FOLDER [--stack ...] [--keep N]
|
||||
#
|
||||
# Config file (default /etc/packetstation/backup.conf, or PS_BACKUP_CONFIG), one stack per line:
|
||||
# # name container database folder
|
||||
# prod project:packetstation packetstation /var/backups/packetstation/prod
|
||||
# test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
# CONTAINER is a container name, or project:<compose project> to find that project's postgres
|
||||
# service by its compose labels. With STACK names given, only those stacks run.
|
||||
# # name database folder
|
||||
# prod packetstation /var/backups/packetstation
|
||||
# With STACK names given, only those stacks run. A VM has one stack; install.sh writes its line.
|
||||
#
|
||||
# pg_dump connects inside the container as the postgres superuser over the local socket, which
|
||||
# the official image trusts. No password is read, passed or printed.
|
||||
# pg_dump and pg_dumpall run as the postgres OS user (setpriv, no PAM) on the local socket, where
|
||||
# PostgreSQL knows that user by peer authentication. No password is read, passed or printed. Their
|
||||
# output goes to files root opened, so the postgres user needs no access to the folder.
|
||||
# PG_BINDIR (default /usr/lib/postgresql/16/bin) picks the client tools, PG_AS the user switch.
|
||||
#
|
||||
# Globals: --no-role-passwords, so no password hash ever lands in a file (or in the PBS copy of
|
||||
# it). The roles' passwords come from the stack's secrets: initdb (deploy/postgres/initdb)
|
||||
# recreates them on a fresh volume. The globals file is also 0600, like the dumps.
|
||||
# it). The role passwords are in /etc/packetstation/secrets; deploy/vm/install.sh sets them.
|
||||
#
|
||||
# Files: dumps and globals are 0600. The markers are 0644 (a time and a file name, nothing else)
|
||||
# so the worker, which mounts the folder read-only as uid 10001, can read them. When run as root
|
||||
# the new files take the folder's owner and group, so a stack folder owned by spencerm stays
|
||||
# readable by spencerm.
|
||||
# so the worker, which reaches the folder through its group, can read them. When run as root the
|
||||
# new files take the folder's owner and group.
|
||||
#
|
||||
# Not in the dump: the stored files (/var/lib/packetstation/assets), the env file and the secrets
|
||||
# (/etc/packetstation). The PBS backup of the VM covers them. See docs/DEPLOY.md.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
KEEP="${KEEP:-7}"
|
||||
PG_USER="${PG_USER:-postgres}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
|
||||
# The command prefix that runs a client tool as the postgres OS user, split on spaces.
|
||||
read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"
|
||||
|
||||
log() { printf '%s ps-backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||||
sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
@@ -53,7 +55,7 @@ usage() {
|
||||
config="${PS_BACKUP_CONFIG:-/etc/packetstation/backup.conf}"
|
||||
config_given=0
|
||||
declare -a only=()
|
||||
declare -a stacks=() # each entry: "name|container|database|folder"
|
||||
declare -a stacks=() # each entry: "name|database|folder"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -61,8 +63,8 @@ while [[ $# -gt 0 ]]; do
|
||||
[[ $# -ge 2 ]] || usage
|
||||
config="$2"; config_given=1; shift 2 ;;
|
||||
--stack)
|
||||
[[ $# -ge 5 ]] || usage
|
||||
stacks+=("$2|$3|$4|$5"); shift 5 ;;
|
||||
[[ $# -ge 4 ]] || usage
|
||||
stacks+=("$2|$3|$4"); shift 4 ;;
|
||||
--keep)
|
||||
[[ $# -ge 2 ]] || usage
|
||||
KEEP="$2"; shift 2 ;;
|
||||
@@ -82,13 +84,13 @@ if [[ ${#stacks[@]} -eq 0 ]]; then
|
||||
log "no --stack given and the config $config is missing or unreadable"
|
||||
exit 2
|
||||
fi
|
||||
while read -r name container database folder extra; do
|
||||
while read -r name database folder extra; do
|
||||
[[ -z "${name:-}" || "${name:0:1}" == "#" ]] && continue
|
||||
if [[ -z "${folder:-}" || -n "${extra:-}" ]]; then
|
||||
log "config line for '$name' needs exactly: name container database folder"
|
||||
log "config line for '$name' needs exactly: name database folder"
|
||||
exit 2
|
||||
fi
|
||||
stacks+=("$name|$container|$database|$folder")
|
||||
stacks+=("$name|$database|$folder")
|
||||
done < "$config"
|
||||
elif [[ $config_given -eq 1 ]]; then
|
||||
log "use --config or --stack, not both"
|
||||
@@ -102,26 +104,9 @@ fi
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
# Resolve "project:<name>" to the running postgres container of that compose project.
|
||||
resolve_container() {
|
||||
local spec="$1"
|
||||
if [[ "$spec" == project:* ]]; then
|
||||
local project="${spec#project:}" ids
|
||||
ids="$("$DOCKER" ps -q \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=postgres")"
|
||||
if [[ -z "$ids" ]]; then
|
||||
echo "no running postgres container for compose project $project" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(printf '%s\n' "$ids" | wc -l)" -ne 1 ]]; then
|
||||
echo "more than one postgres container for compose project $project" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$ids"
|
||||
else
|
||||
printf '%s\n' "$spec"
|
||||
fi
|
||||
# A PostgreSQL client tool: from PG_BINDIR when it is set, else from PATH.
|
||||
pgbin() {
|
||||
if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
|
||||
}
|
||||
|
||||
# Write a marker atomically: temp file in the same folder, then rename.
|
||||
@@ -203,8 +188,8 @@ on_stack_exit() {
|
||||
}
|
||||
|
||||
backup_stack() {
|
||||
local name="$1" spec="$2" database="$3" folder="$4"
|
||||
local stamp globals_name container started entries size
|
||||
local name="$1" database="$2" folder="$3"
|
||||
local stamp globals_name started entries size
|
||||
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
cur_dump_name="ps-$stamp.dump"
|
||||
@@ -213,34 +198,33 @@ backup_stack() {
|
||||
log "stack $name: database $database into $folder"
|
||||
|
||||
if [[ ! -d "$folder" ]]; then
|
||||
# 0750: the worker reaches the folder through its group (see scripts/backup/README.md).
|
||||
# 0750: the worker reaches the folder through its group (docs/DEPLOY.md, "Backups on a VM").
|
||||
mkdir -p -- "$folder"
|
||||
chmod 0750 -- "$folder" 2>/dev/null || log "stack $name: could not set mode 0750 on $folder"
|
||||
fi
|
||||
cur_folder="$folder"
|
||||
[[ -w "$folder" ]] || fail_stack "$folder is not writable"
|
||||
[[ "$database" =~ ^[A-Za-z_][A-Za-z0-9_]{0,62}$ ]] || fail_stack "database name '$database' is not a plain name"
|
||||
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
exec 9>"$folder/.ps-backup.lock"
|
||||
flock -n 9 || fail_stack "another backup of this stack is running"
|
||||
fi
|
||||
|
||||
container="$(resolve_container "$spec" 2>&1)" || fail_stack "$container"
|
||||
|
||||
cur_tmp_dump="$(mktemp "$folder/.$cur_dump_name.XXXXXX.partial")"
|
||||
cur_tmp_globals="$(mktemp "$folder/.$globals_name.XXXXXX.partial")"
|
||||
|
||||
"$DOCKER" exec "$container" pg_dump -U "$PG_USER" -d "$database" -Fc > "$cur_tmp_dump" \
|
||||
"${PG_AS[@]}" "$(pgbin pg_dump)" -d "$database" -Fc > "$cur_tmp_dump" \
|
||||
|| fail_stack "pg_dump failed"
|
||||
[[ -s "$cur_tmp_dump" ]] || fail_stack "pg_dump wrote nothing"
|
||||
|
||||
"$DOCKER" exec "$container" pg_dumpall -U "$PG_USER" --globals-only --no-role-passwords \
|
||||
"${PG_AS[@]}" "$(pgbin pg_dumpall)" --globals-only --no-role-passwords \
|
||||
> "$cur_tmp_globals" || fail_stack "pg_dumpall --globals-only failed"
|
||||
[[ -s "$cur_tmp_globals" ]] || fail_stack "pg_dumpall wrote nothing"
|
||||
|
||||
# The check reads the dump from stdin with the container's own pg_restore. Entry lines start
|
||||
# with the entry's id; comment lines start with a semicolon.
|
||||
entries="$("$DOCKER" exec -i "$container" pg_restore --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
|
||||
# The check reads the dump from stdin and needs no database. Entry lines start with the
|
||||
# entry's id; comment lines start with a semicolon.
|
||||
entries="$("$(pgbin pg_restore)" --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
|
||||
|| fail_stack "pg_restore --list could not read the dump"
|
||||
[[ "$entries" -ge 1 ]] || fail_stack "pg_restore --list found no entries"
|
||||
|
||||
@@ -269,7 +253,7 @@ run_stack() (
|
||||
failures=0
|
||||
ran=0
|
||||
for entry in "${stacks[@]}"; do
|
||||
IFS='|' read -r s_name s_container s_database s_folder <<< "$entry"
|
||||
IFS='|' read -r s_name s_database s_folder <<< "$entry"
|
||||
if [[ ${#only[@]} -gt 0 ]]; then
|
||||
wanted=0
|
||||
for o in "${only[@]}"; do
|
||||
@@ -280,7 +264,7 @@ for entry in "${stacks[@]}"; do
|
||||
ran=$(( ran + 1 ))
|
||||
# Not in an if or ||: that would switch errexit off inside the subshell.
|
||||
set +e
|
||||
run_stack "$s_name" "$s_container" "$s_database" "$s_folder"
|
||||
run_stack "$s_name" "$s_database" "$s_folder"
|
||||
rc=$?
|
||||
set -e
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
|
||||
@@ -1,75 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# Packet Station restore. Runs on the docker host. Restores a ps-backup.sh dump into a fresh
|
||||
# database on a stack's postgres container, optionally applies the globals file, copies assets
|
||||
# back into an assets volume, and prints the row count of every table at the end.
|
||||
# Packet Station restore on a native VM. Run by root. Restores a ps-backup dump into a fresh
|
||||
# database, optionally applies the globals file, copies stored files back into an assets folder,
|
||||
# checks them against the database, and prints the row count of every table at the end.
|
||||
#
|
||||
# It never stops, restarts or reconfigures a stack. Pointing the app at the restored database
|
||||
# (or renaming it into place) is a separate, deliberate step: see scripts/backup/README.md.
|
||||
# It never stops, starts or reconfigures a service. Putting the restored database into service
|
||||
# (renaming it into place) is a separate, deliberate step: see docs/DEPLOY.md, "Restore on a VM".
|
||||
#
|
||||
# Usage:
|
||||
# ps-restore.sh --container NAME | --project PROJECT
|
||||
# [--dump FILE --db NAME [--replace]]
|
||||
# ps-restore [--dump FILE --db NAME [--replace]]
|
||||
# [--globals FILE]
|
||||
# [--assets DIR_OR_TAR --assets-volume VOLUME]
|
||||
# [--verify-assets --db NAME --assets-volume VOLUME]
|
||||
# [--assets DIR_OR_TAR --assets-dir DIR]
|
||||
# [--verify-assets --db NAME --assets-dir DIR]
|
||||
# [--counts-only --db NAME]
|
||||
#
|
||||
# --container / --project the postgres container, by name or by compose project.
|
||||
# --dump FILE a ps-*.dump file (pg_dump -Fc).
|
||||
# --db NAME the database to restore into. It must not exist, unless --replace,
|
||||
# which drops it first. DROP DATABASE refuses while anyone is connected.
|
||||
# --owner ROLE owner of the new database (default ps_migrate, as initdb makes it).
|
||||
# --replace is refused for packetstation, the live database.
|
||||
# --owner ROLE owner of the new database (default ps_migrate, as install.sh makes it).
|
||||
# --app-role ROLE role given CONNECT and TEMPORARY on it (default ps_app).
|
||||
# --locale NAME the new database's locale (default en_US.UTF-8, as install.sh makes it).
|
||||
# --globals FILE a ps-*.globals.sql file. Roles that already exist are reported and
|
||||
# left alone. The file holds no passwords: set them from the secrets.
|
||||
# left alone. The file holds no passwords: install.sh sets them again.
|
||||
# --assets SRC a folder, or a .tar / .tar.gz / .tgz, holding the files that belong
|
||||
# under ASSETS_DIR. Copied into --assets-volume through a throwaway
|
||||
# container (no network) that runs the postgres image already on the host.
|
||||
# Existing files are never overwritten.
|
||||
# --assets-volume VOLUME the docker volume to copy into (for example packetstation-test_assets).
|
||||
# --assets-owner UID:GID owner for the copied files (default 10001:10001, the image's ps user).
|
||||
# --verify-assets compare sha256 of every file in --assets-volume with assets.sha256 in --db.
|
||||
# under ASSETS_DIR. Existing files are never overwritten.
|
||||
# --assets-dir DIR the folder to copy into or check (/var/lib/packetstation/assets is the
|
||||
# live one; a scratch folder for a rehearsal).
|
||||
# --assets-owner U:G owner for the copied files (default packetstation:packetstation).
|
||||
# --verify-assets compare sha256 of every file in --assets-dir with assets.sha256 in --db.
|
||||
# --counts-only only print the row counts of --db.
|
||||
#
|
||||
# Connects inside the container as the postgres superuser over the local socket. No password
|
||||
# is read, passed or printed.
|
||||
# psql and pg_restore run as the postgres OS user on the local socket (peer authentication). The
|
||||
# files are opened by root and handed over on stdin. No password is read, passed or printed.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
PG_USER="${PG_USER:-postgres}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
|
||||
read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"
|
||||
LIVE_DB=packetstation
|
||||
|
||||
log() { printf '%s ps-restore: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
|
||||
die() { log "$*"; exit 1; }
|
||||
usage() { sed -n '2,36p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
usage() { sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
|
||||
container=""
|
||||
project=""
|
||||
dump=""
|
||||
db=""
|
||||
replace=0
|
||||
owner="ps_migrate"
|
||||
app_role="ps_app"
|
||||
locale_name="en_US.UTF-8"
|
||||
globals=""
|
||||
assets=""
|
||||
assets_volume=""
|
||||
assets_owner="10001:10001"
|
||||
assets_dir=""
|
||||
assets_owner="packetstation:packetstation"
|
||||
verify_assets=0
|
||||
counts_only=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--container) container="${2:?}"; shift 2 ;;
|
||||
--project) project="${2:?}"; shift 2 ;;
|
||||
--dump) dump="${2:?}"; shift 2 ;;
|
||||
--db) db="${2:?}"; shift 2 ;;
|
||||
--replace) replace=1; shift ;;
|
||||
--owner) owner="${2:?}"; shift 2 ;;
|
||||
--app-role) app_role="${2:?}"; shift 2 ;;
|
||||
--locale) locale_name="${2:?}"; shift 2 ;;
|
||||
--globals) globals="${2:?}"; shift 2 ;;
|
||||
--assets) assets="${2:?}"; shift 2 ;;
|
||||
--assets-volume) assets_volume="${2:?}"; shift 2 ;;
|
||||
--assets-dir) assets_dir="${2:?}"; shift 2 ;;
|
||||
--assets-owner) assets_owner="${2:?}"; shift 2 ;;
|
||||
--verify-assets) verify_assets=1; shift ;;
|
||||
--counts-only) counts_only=1; shift ;;
|
||||
@@ -82,32 +80,31 @@ ident_re='^[a-z_][a-z0-9_]{0,62}$'
|
||||
[[ -z "$db" || "$db" =~ $ident_re ]] || die "--db must be a plain lower-case name (letters, digits, _)"
|
||||
[[ "$owner" =~ $ident_re ]] || die "--owner must be a plain lower-case role name"
|
||||
[[ "$app_role" =~ $ident_re ]] || die "--app-role must be a plain lower-case role name"
|
||||
[[ "$assets_owner" =~ ^[0-9]+:[0-9]+$ ]] || die "--assets-owner must be UID:GID"
|
||||
|
||||
if [[ -n "$project" ]]; then
|
||||
[[ -z "$container" ]] || die "use --container or --project, not both"
|
||||
container="$("$DOCKER" ps -q \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=postgres")"
|
||||
[[ -n "$container" ]] || die "no running postgres container for compose project $project"
|
||||
[[ "$(printf '%s\n' "$container" | wc -l)" -eq 1 ]] || die "more than one postgres container for $project"
|
||||
fi
|
||||
[[ -n "$container" ]] || die "--container or --project is required"
|
||||
[[ "$locale_name" =~ ^[A-Za-z0-9_.@-]+$ ]] || die "--locale must be a locale name like en_US.UTF-8"
|
||||
[[ "$assets_owner" =~ ^[a-z_0-9][a-z0-9_-]*:[a-z_0-9][a-z0-9_-]*$ ]] || die "--assets-owner must be USER:GROUP"
|
||||
|
||||
if [[ -n "$dump" || $counts_only -eq 1 || $verify_assets -eq 1 ]]; then
|
||||
[[ -n "$db" ]] || die "--db is required"
|
||||
fi
|
||||
if [[ -n "$assets" || $verify_assets -eq 1 ]]; then
|
||||
[[ -n "$assets_volume" ]] || die "--assets-volume is required"
|
||||
[[ -n "$assets_dir" ]] || die "--assets-dir is required"
|
||||
[[ -d "$assets_dir" ]] || die "$assets_dir is not a folder"
|
||||
fi
|
||||
if [[ -z "$dump" && -z "$globals" && -z "$assets" && $verify_assets -eq 0 && $counts_only -eq 0 ]]; then
|
||||
die "nothing to do: give --dump, --globals, --assets, --verify-assets or --counts-only"
|
||||
fi
|
||||
if [[ -n "$dump" && $replace -eq 1 && "$db" == "$LIVE_DB" ]]; then
|
||||
die "--replace is refused for $LIVE_DB. Restore into another name and rename it into place (docs/DEPLOY.md)."
|
||||
fi
|
||||
|
||||
# psql inside the container, reading SQL from stdin. Variables go through -v and :'name'.
|
||||
pgbin() {
|
||||
if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
|
||||
}
|
||||
|
||||
# psql as postgres, reading SQL from stdin. Variables go through -v and :'name'.
|
||||
psql_c() {
|
||||
local dbname="$1"; shift
|
||||
"$DOCKER" exec -i "$container" psql -X -q -v ON_ERROR_STOP=1 -U "$PG_USER" -d "$dbname" "$@"
|
||||
"${PG_AS[@]}" "$(pgbin psql)" -X -q -v ON_ERROR_STOP=1 -d "$dbname" "$@"
|
||||
}
|
||||
|
||||
db_exists() {
|
||||
@@ -143,22 +140,16 @@ ORDER BY c.relname;
|
||||
SQL
|
||||
}
|
||||
|
||||
# A throwaway container on no network, running the image the postgres container already uses,
|
||||
# so nothing is downloaded.
|
||||
helper_image() {
|
||||
"$DOCKER" inspect -f '{{.Config.Image}}' "$container"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ globals
|
||||
|
||||
if [[ -n "$globals" ]]; then
|
||||
[[ -r "$globals" ]] || die "cannot read $globals"
|
||||
if grep -qiE "PASSWORD[[:space:]]+'(SCRAM|md5)" "$globals"; then
|
||||
die "$globals holds password hashes; ps-backup.sh writes globals without them. Refusing."
|
||||
die "$globals holds password hashes; ps-backup writes globals without them. Refusing."
|
||||
fi
|
||||
log "applying globals from $(basename -- "$globals") (existing roles are left as they are)"
|
||||
# Without ON_ERROR_STOP: CREATE ROLE for a role that exists fails and the rest goes on.
|
||||
out="$("$DOCKER" exec -i "$container" psql -X -q -U "$PG_USER" -d postgres < "$globals" 2>&1 || true)"
|
||||
out="$("${PG_AS[@]}" "$(pgbin psql)" -X -q -d postgres < "$globals" 2>&1 || true)"
|
||||
exists="$(printf '%s\n' "$out" | grep -c 'already exists' || true)"
|
||||
other="$(printf '%s\n' "$out" | grep 'ERROR' | grep -vc 'already exists' || true)"
|
||||
log "globals: $exists statements skipped (already exists), $other other errors"
|
||||
@@ -180,12 +171,13 @@ if [[ -n "$dump" ]]; then
|
||||
SELECT format('DROP DATABASE %I', :'name') \gexec
|
||||
SQL
|
||||
fi
|
||||
role_exists "$owner" || die "role $owner does not exist (apply --globals or start the stack once)"
|
||||
role_exists "$owner" || die "role $owner does not exist (apply --globals, or run deploy/vm/install.sh)"
|
||||
|
||||
log "creating database $db owned by $owner"
|
||||
# The same shape as deploy/postgres/initdb/01-roles.sh.
|
||||
psql_c postgres -v name="$db" -v owner="$owner" <<'SQL'
|
||||
SELECT format('CREATE DATABASE %I OWNER %I ENCODING %L TEMPLATE template0', :'name', :'owner', 'UTF8') \gexec
|
||||
# The same shape as deploy/vm/install.sh makes the live one.
|
||||
psql_c postgres -v name="$db" -v owner="$owner" -v loc="$locale_name" <<'SQL'
|
||||
SELECT format('CREATE DATABASE %I OWNER %I ENCODING %L LOCALE %L TEMPLATE template0',
|
||||
:'name', :'owner', 'UTF8', :'loc') \gexec
|
||||
SELECT format('REVOKE ALL ON DATABASE %I FROM PUBLIC', :'name') \gexec
|
||||
SQL
|
||||
if role_exists "$app_role"; then
|
||||
@@ -198,8 +190,8 @@ SQL
|
||||
|
||||
log "restoring $(basename -- "$dump") ($(stat -c %s "$dump") bytes) into $db"
|
||||
# One transaction: a restore that fails leaves an empty database, never a half-restored one.
|
||||
"$DOCKER" exec -i "$container" pg_restore -U "$PG_USER" -d "$db" --exit-on-error --single-transaction \
|
||||
< "$dump" || die "pg_restore failed; database $db is left empty. Drop it with --replace on the next try."
|
||||
"${PG_AS[@]}" "$(pgbin pg_restore)" -d "$db" --exit-on-error --single-transaction < "$dump" \
|
||||
|| die "pg_restore failed; database $db is left empty. Drop it with --replace on the next try."
|
||||
log "database restored in $(( SECONDS - started )) s"
|
||||
fi
|
||||
|
||||
@@ -207,26 +199,21 @@ fi
|
||||
|
||||
if [[ -n "$assets" ]]; then
|
||||
[[ -e "$assets" ]] || die "$assets does not exist"
|
||||
"$DOCKER" volume inspect "$assets_volume" >/dev/null 2>&1 || die "volume $assets_volume does not exist"
|
||||
image="$(helper_image)"
|
||||
started=$SECONDS
|
||||
src="$(cd -- "$(dirname -- "$assets")" && pwd)/$(basename -- "$assets")"
|
||||
if [[ -d "$assets" ]]; then
|
||||
log "copying assets from folder $src into volume $assets_volume"
|
||||
"$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
|
||||
-v "$assets_volume:/dst" -v "$src:/src:ro" "$image" -c \
|
||||
'set -eu; cp -a -n /src/. /dst/; chown -R "$1" /dst' sh "$assets_owner"
|
||||
log "copying assets from folder $assets into $assets_dir"
|
||||
cp -a -n -- "$assets/." "$assets_dir/"
|
||||
else
|
||||
case "$assets" in
|
||||
*.tar.gz|*.tgz) tar_flags="-xzf" ;;
|
||||
*.tar) tar_flags="-xf" ;;
|
||||
*) die "--assets must be a folder, .tar, .tar.gz or .tgz" ;;
|
||||
esac
|
||||
log "copying assets from $(basename -- "$src") into volume $assets_volume"
|
||||
"$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
|
||||
-v "$assets_volume:/dst" -v "$src:/src.tar:ro" "$image" -c \
|
||||
'set -eu; tar "$1" /src.tar -C /dst --skip-old-files --no-same-owner; chown -R "$2" /dst' \
|
||||
sh "$tar_flags" "$assets_owner"
|
||||
log "copying assets from $(basename -- "$assets") into $assets_dir"
|
||||
tar "$tar_flags" "$assets" -C "$assets_dir" --skip-old-files --no-same-owner
|
||||
fi
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
chown -R -- "$assets_owner" "$assets_dir"
|
||||
fi
|
||||
log "assets copied in $(( SECONDS - started )) s"
|
||||
fi
|
||||
@@ -234,8 +221,6 @@ fi
|
||||
# ------------------------------------------------------------------ verify assets
|
||||
|
||||
if [[ $verify_assets -eq 1 ]]; then
|
||||
"$DOCKER" volume inspect "$assets_volume" >/dev/null 2>&1 || die "volume $assets_volume does not exist"
|
||||
image="$(helper_image)"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
# sha256sum check format: "<hex> <path>".
|
||||
@@ -243,21 +228,11 @@ if [[ $verify_assets -eq 1 ]]; then
|
||||
SELECT encode(sha256, 'hex') || ' ' || storage_key FROM assets ORDER BY storage_key;
|
||||
SQL
|
||||
rows="$(wc -l < "$work/assets.sha256")"
|
||||
chmod 0755 "$work"
|
||||
chmod 0644 "$work/assets.sha256"
|
||||
log "checking $rows asset rows of $db against volume $assets_volume"
|
||||
set +e
|
||||
"$DOCKER" run --rm --network none --user 0:0 --entrypoint sh \
|
||||
-v "$assets_volume:/dst:ro" -v "$work:/work:ro" "$image" -c '
|
||||
cd /dst || exit 3
|
||||
sha256sum -c /work/assets.sha256 2>/dev/null
|
||||
find . -type f ! -path "./tmp/*" | sed "s|^\./||" | sort > /tmp/files
|
||||
cut -c67- /work/assets.sha256 | sort > /tmp/rows
|
||||
comm -23 /tmp/files /tmp/rows | sed "s|^|ORPHAN: |"
|
||||
exit 0' > "$work/result"
|
||||
rc=$?
|
||||
set -e
|
||||
[[ $rc -eq 0 ]] || die "the check container failed (exit $rc)"
|
||||
log "checking $rows asset rows of $db against $assets_dir"
|
||||
( cd -- "$assets_dir" && sha256sum -c "$work/assets.sha256" 2>/dev/null ) > "$work/result" || true
|
||||
( cd -- "$assets_dir" && find . -type f ! -path './tmp/*' | sed 's|^\./||' | LC_ALL=C sort ) > "$work/files"
|
||||
cut -c67- "$work/assets.sha256" | LC_ALL=C sort > "$work/rows"
|
||||
LC_ALL=C comm -23 "$work/files" "$work/rows" | sed 's|^|ORPHAN: |' >> "$work/result"
|
||||
ok="$(grep -c ': OK$' "$work/result" || true)"
|
||||
bad="$(grep -c ': FAILED$' "$work/result" || true)"
|
||||
missing="$(grep -c ': FAILED open or read$' "$work/result" || true)"
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
"""scripts/backup/ps-backup.sh with a fake `docker` on PATH.
|
||||
"""The VM backup and restore scripts (scripts/backup) and the VM kit's shell parts (deploy/vm).
|
||||
|
||||
No Docker and no database: the fake answers `docker ps`, `docker exec ... pg_dump`, `pg_dumpall` and
|
||||
`pg_restore --list`, and records every call. Runs on Linux (CI, docker-1) and on Windows when a
|
||||
Git Bash or Cygwin bash with cygpath is installed; skipped otherwise.
|
||||
No PostgreSQL, no systemd and no root: fake `pg_dump`, `pg_dumpall`, `pg_restore` and `psql` sit in a
|
||||
folder given as PG_BINDIR, a fake user switch stands in for setpriv (PG_AS), and a fake `systemd-run`
|
||||
prints what ps-run would start. Every fake records its calls. Runs on Linux (CI, docker-1) and on
|
||||
Windows when a Git Bash or Cygwin bash with cygpath is installed; skipped otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
@@ -20,29 +22,23 @@ import pytest
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
BACKUP = REPO / "scripts" / "backup" / "ps-backup.sh"
|
||||
RESTORE = REPO / "scripts" / "backup" / "ps-restore.sh"
|
||||
VM = REPO / "deploy" / "vm"
|
||||
UNITS = VM / "systemd"
|
||||
WINDOWS = sys.platform == "win32"
|
||||
|
||||
FAKE_DOCKER = r"""#!/usr/bin/env bash
|
||||
# Fake docker for the backup script tests. Behaviour comes from FAKE_* variables.
|
||||
printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG"
|
||||
cmd="$1"; shift
|
||||
case "$cmd" in
|
||||
ps)
|
||||
if [[ -n "${FAKE_PS_IDS+x}" ]]; then
|
||||
[[ -n "$FAKE_PS_IDS" ]] && printf '%s\n' $FAKE_PS_IDS
|
||||
else
|
||||
echo pgfake
|
||||
fi
|
||||
exit 0 ;;
|
||||
exec)
|
||||
[[ "$1" == "-i" ]] && shift
|
||||
container="$1"; shift
|
||||
tool="$1"; shift
|
||||
# One script, written under each tool's name. It dispatches on its own name.
|
||||
FAKE_PG = r"""#!/usr/bin/env bash
|
||||
tool="$(basename "$0")"
|
||||
printf '%s %s\n' "$tool" "$*" >> "$FAKE_LOG"
|
||||
case "$tool" in
|
||||
asuser)
|
||||
exec "$@" ;;
|
||||
pg_dump)
|
||||
[[ "$container" == "${FAKE_DUMP_FAIL_CONTAINER:-none}" ]] && { echo "pg_dump: error: fake" >&2; exit 1; }
|
||||
db=""
|
||||
while [[ $# -gt 0 ]]; do [[ "$1" == "-d" ]] && db="$2"; shift; done
|
||||
[[ "$db" == "${FAKE_DUMP_FAIL_DB:-none}" ]] && { echo "pg_dump: error: fake" >&2; exit 1; }
|
||||
[[ -n "${FAKE_DUMP_EMPTY:-}" ]] && exit 0
|
||||
printf 'PGDMP fake dump of %s\n' "$container"
|
||||
printf 'PGDMP fake dump of %s\n' "$db"
|
||||
exit 0 ;;
|
||||
pg_dumpall)
|
||||
printf 'CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN;\n'
|
||||
@@ -51,15 +47,40 @@ case "$cmd" in
|
||||
input="$(cat)"
|
||||
[[ -n "${FAKE_RESTORE_FAIL:-}" ]] && { echo "pg_restore: error: input file is too short" >&2; exit 1; }
|
||||
[[ "$input" == PGDMP* ]] || { echo "pg_restore: error: not a dump" >&2; exit 1; }
|
||||
if [[ " $* " == *" --list "* ]]; then
|
||||
printf ';\n; Archive created at 2026-10-21\n;\n215; 1259 16390 TABLE public packets ps_migrate\n'
|
||||
printf '3801; 0 16390 TABLE DATA public packets ps_migrate\n'
|
||||
fi
|
||||
exit 0 ;;
|
||||
esac ;;
|
||||
psql)
|
||||
sql="$(cat)"
|
||||
printf '%s\n' "$sql" | sed 's/^/ sql: /' >> "$FAKE_LOG"
|
||||
case "$sql" in
|
||||
*"FROM pg_database"*) echo "${FAKE_DB_EXISTS:-0}" ;;
|
||||
*"FROM pg_roles"*) echo "${FAKE_ROLE_EXISTS:-1}" ;;
|
||||
*"FROM assets"*) cat "$FAKE_ASSET_ROWS" ;;
|
||||
*"relkind"*) printf 'packets\t3\n' ;;
|
||||
*"CREATE ROLE"*) printf '%s\n' "${FAKE_GLOBALS_ERR:-}" >&2 ;;
|
||||
esac
|
||||
echo "fake docker: unexpected call: $cmd $*" >&2
|
||||
exit 0 ;;
|
||||
esac
|
||||
echo "fake: unexpected tool $tool" >&2
|
||||
exit 99
|
||||
"""
|
||||
|
||||
FAKE_SYSTEMD_RUN = r"""#!/usr/bin/env bash
|
||||
for a in "$@"; do printf '%s\n' "$a"; done
|
||||
"""
|
||||
|
||||
FAKE_PACKETSTATION = r"""#!/usr/bin/env bash
|
||||
printf 'args=%s\n' "$*"
|
||||
for v in PG_APP_PASSWORD_FILE PG_MIGRATE_PASSWORD_FILE VISUAL_PASSWORD_FILE MFILES_PASSWORD_FILE \
|
||||
SESSION_KEY_FILE SEAL_KEY_FILE STATIC_DIR PS_BUILD WORKER_BEAT_FILE TZ; do
|
||||
printf '%s=%s\n' "$v" "${!v-<unset>}"
|
||||
done
|
||||
printf 'MFILES_PASSWORD=%s\n' "${MFILES_PASSWORD-<unset>}"
|
||||
"""
|
||||
|
||||
|
||||
def _find_bash() -> str | None:
|
||||
if not WINDOWS:
|
||||
@@ -74,7 +95,7 @@ def _find_bash() -> str | None:
|
||||
continue
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[c, "-c", "cygpath -u 'C:\\' && command -v find stat mktemp"],
|
||||
[c, "-c", "cygpath -u 'C:\\' && command -v find stat mktemp sha256sum"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
@@ -87,7 +108,7 @@ def _find_bash() -> str | None:
|
||||
|
||||
|
||||
BASH = _find_bash()
|
||||
pytestmark = pytest.mark.skipif(BASH is None, reason="no usable bash (Linux bash, or Git Bash with cygpath)")
|
||||
needs_bash = pytest.mark.skipif(BASH is None, reason="no usable bash (Linux bash, or Git Bash with cygpath)")
|
||||
|
||||
|
||||
def _posix(p: Path) -> str:
|
||||
@@ -98,33 +119,45 @@ def _posix(p: Path) -> str:
|
||||
return out.stdout.strip()
|
||||
|
||||
|
||||
def _write_exec(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fake(tmp_path: Path) -> dict[str, object]:
|
||||
bindir = tmp_path / "bin"
|
||||
bindir = tmp_path / "pgbin"
|
||||
bindir.mkdir()
|
||||
docker = bindir / "docker"
|
||||
docker.write_text(FAKE_DOCKER, encoding="utf-8", newline="\n")
|
||||
docker.chmod(docker.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
log = tmp_path / "docker.log"
|
||||
for tool in ("asuser", "pg_dump", "pg_dumpall", "pg_restore", "psql"):
|
||||
_write_exec(bindir / tool, FAKE_PG)
|
||||
log = tmp_path / "calls.log"
|
||||
log.touch()
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith("FAKE_")}
|
||||
env["FAKE_DOCKER_LOG"] = _posix(log)
|
||||
env["DOCKER"] = _posix(docker)
|
||||
env["PATH"] = str(bindir) + os.pathsep + env.get("PATH", "")
|
||||
env.pop("KEEP", None)
|
||||
env.pop("PS_BACKUP_CONFIG", None)
|
||||
env["FAKE_LOG"] = _posix(log)
|
||||
env["PG_BINDIR"] = _posix(bindir)
|
||||
env["PG_AS"] = _posix(bindir / "asuser")
|
||||
for k in ("KEEP", "PS_BACKUP_CONFIG"):
|
||||
env.pop(k, None)
|
||||
return {"env": env, "log": log, "tmp": tmp_path}
|
||||
|
||||
|
||||
def run_backup(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
|
||||
def _run(script: Path, fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
|
||||
env = dict(fake["env"]) # type: ignore[call-overload]
|
||||
env.update(extra_env)
|
||||
assert BASH is not None
|
||||
return subprocess.run([BASH, _posix(BACKUP), *args], capture_output=True, text=True, env=env, timeout=120)
|
||||
return subprocess.run([BASH, _posix(script), *args], capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
def stack_args(name: str, folder: Path, container: str = "project:packetstation-test") -> list[str]:
|
||||
return ["--stack", name, container, "packetstation", _posix(folder)]
|
||||
def run_backup(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
|
||||
return _run(BACKUP, fake, *args, **extra_env)
|
||||
|
||||
|
||||
def run_restore(fake: dict[str, object], *args: str, **extra_env: str) -> subprocess.CompletedProcess[str]:
|
||||
return _run(RESTORE, fake, *args, **extra_env)
|
||||
|
||||
|
||||
def stack_args(name: str, folder: Path, database: str = "packetstation") -> list[str]:
|
||||
return ["--stack", name, database, _posix(folder)]
|
||||
|
||||
|
||||
def dumps(folder: Path) -> list[str]:
|
||||
@@ -138,6 +171,10 @@ def no_partials(folder: Path) -> bool:
|
||||
MARKER = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z (ps-\d{8}T\d{6}Z\.dump)(.*)$")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- ps-backup
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_success_writes_dump_globals_and_marker(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
res = run_backup(fake, *stack_args("test", folder))
|
||||
@@ -157,12 +194,14 @@ def test_success_writes_dump_globals_and_marker(fake: dict[str, object]) -> None
|
||||
assert stat.S_IMODE(globals_file.stat().st_mode) == 0o600
|
||||
assert stat.S_IMODE((folder / "last-success").stat().st_mode) == 0o644
|
||||
calls = fake["log"].read_text() # type: ignore[attr-defined]
|
||||
assert "label=com.docker.compose.project=packetstation-test" in calls
|
||||
assert "exec pgfake pg_dump -U postgres -d packetstation -Fc" in calls
|
||||
assert "pg_dumpall -U postgres --globals-only --no-role-passwords" in calls
|
||||
assert "exec -i pgfake pg_restore --list" in calls
|
||||
# pg_dump and pg_dumpall go through the user switch; pg_restore --list needs no database and no switch.
|
||||
assert re.search(r"^asuser \S*/pg_dump -d packetstation -Fc$", calls, re.M), calls
|
||||
assert re.search(r"^asuser \S*/pg_dumpall --globals-only --no-role-passwords$", calls, re.M), calls
|
||||
assert re.search(r"^pg_restore --list$", calls, re.M), calls
|
||||
assert "docker" not in calls
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_rotation_keeps_seven(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
folder.mkdir()
|
||||
@@ -183,6 +222,7 @@ def test_rotation_keeps_seven(fake: dict[str, object]) -> None:
|
||||
assert sorted(p.name for p in folder.glob("*.globals.sql")) == [n.replace(".dump", ".globals.sql") for n in names]
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_keep_is_configurable(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
folder.mkdir()
|
||||
@@ -191,8 +231,10 @@ def test_keep_is_configurable(fake: dict[str, object]) -> None:
|
||||
res = run_backup(fake, "--keep", "2", *stack_args("test", folder))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert len(dumps(folder)) == 2
|
||||
assert run_backup(fake, "--keep", "0", *stack_args("test", folder)).returncode == 2
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_failed_restore_list_writes_last_failure(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
folder.mkdir()
|
||||
@@ -212,39 +254,39 @@ def test_failed_restore_list_writes_last_failure(fake: dict[str, object]) -> Non
|
||||
assert "FAILED" in res.stderr
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_failed_pg_dump_and_empty_dump(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
res = run_backup(fake, *stack_args("test", folder, container="pgdirect"), FAKE_DUMP_FAIL_CONTAINER="pgdirect")
|
||||
res = run_backup(fake, *stack_args("test", folder, database="nosuchdb"), FAKE_DUMP_FAIL_DB="nosuchdb")
|
||||
assert res.returncode != 0
|
||||
assert "pg_dump failed" in (folder / "last-failure").read_text()
|
||||
assert dumps(folder) == []
|
||||
assert no_partials(folder)
|
||||
|
||||
res = run_backup(fake, *stack_args("test", folder, container="pgdirect"), FAKE_DUMP_EMPTY="1")
|
||||
res = run_backup(fake, *stack_args("test", folder), FAKE_DUMP_EMPTY="1")
|
||||
assert res.returncode != 0
|
||||
assert "pg_dump wrote nothing" in (folder / "last-failure").read_text()
|
||||
assert not (folder / "last-success").exists()
|
||||
|
||||
|
||||
def test_no_container_for_project(fake: dict[str, object]) -> None:
|
||||
@needs_bash
|
||||
def test_bad_database_name_is_refused(fake: dict[str, object]) -> None:
|
||||
folder = fake["tmp"] / "backups" # type: ignore[operator]
|
||||
res = run_backup(fake, *stack_args("test", folder), FAKE_PS_IDS="")
|
||||
res = run_backup(fake, *stack_args("test", folder, database="x;rm -rf"))
|
||||
assert res.returncode != 0
|
||||
assert "no running postgres container" in (folder / "last-failure").read_text()
|
||||
|
||||
res = run_backup(fake, *stack_args("test", folder), FAKE_PS_IDS="aaa bbb")
|
||||
assert res.returncode != 0
|
||||
assert "more than one postgres container" in (folder / "last-failure").read_text()
|
||||
assert "not a plain name" in (folder / "last-failure").read_text()
|
||||
assert "pg_dump" not in fake["log"].read_text() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_one_stack_fails_the_other_still_runs(fake: dict[str, object]) -> None:
|
||||
good = fake["tmp"] / "good" # type: ignore[operator]
|
||||
bad = fake["tmp"] / "bad" # type: ignore[operator]
|
||||
res = run_backup(
|
||||
fake,
|
||||
*stack_args("bad", bad, container="pgbad"),
|
||||
*stack_args("good", good, container="pggood"),
|
||||
FAKE_DUMP_FAIL_CONTAINER="pgbad",
|
||||
*stack_args("bad", bad, database="baddb"),
|
||||
*stack_args("good", good),
|
||||
FAKE_DUMP_FAIL_DB="baddb",
|
||||
)
|
||||
assert res.returncode == 1
|
||||
assert (bad / "last-failure").exists() and not (bad / "last-success").exists()
|
||||
@@ -252,15 +294,13 @@ def test_one_stack_fails_the_other_still_runs(fake: dict[str, object]) -> None:
|
||||
assert len(dumps(good)) == 1
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_config_file_and_stack_selection(fake: dict[str, object]) -> None:
|
||||
tmp: Path = fake["tmp"] # type: ignore[assignment]
|
||||
a, b = tmp / "a", tmp / "b"
|
||||
conf = tmp / "backup.conf"
|
||||
conf.write_text(
|
||||
"# name container database folder\n"
|
||||
f"alpha pga packetstation {_posix(a)}\n"
|
||||
"\n"
|
||||
f"beta pgb packetstation {_posix(b)}\n",
|
||||
f"# name database folder\nalpha packetstation {_posix(a)}\n\nbeta packetstation {_posix(b)}\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
@@ -275,13 +315,382 @@ def test_config_file_and_stack_selection(fake: dict[str, object]) -> None:
|
||||
res = run_backup(fake, "--config", _posix(tmp / "missing.conf"))
|
||||
assert res.returncode == 2
|
||||
|
||||
# The old docker-1 format (with a container column) is refused, not misread.
|
||||
conf.write_text(f"test project:packetstation-test packetstation {_posix(a)}\n", encoding="utf-8", newline="\n")
|
||||
res = run_backup(fake, "--config", _posix(conf))
|
||||
assert res.returncode == 2
|
||||
assert "name database folder" in res.stderr
|
||||
|
||||
def test_scripts_are_strict_and_parse() -> None:
|
||||
|
||||
# ---------------------------------------------------------------- ps-restore
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_restore_into_a_fresh_database(fake: dict[str, object]) -> None:
|
||||
tmp: Path = fake["tmp"] # type: ignore[assignment]
|
||||
dump = tmp / "ps-20261030T080000Z.dump"
|
||||
dump.write_text("PGDMP fake")
|
||||
res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored")
|
||||
assert res.returncode == 0, res.stderr
|
||||
calls = fake["log"].read_text() # type: ignore[attr-defined]
|
||||
assert "CREATE DATABASE %I OWNER %I ENCODING %L LOCALE %L TEMPLATE template0" in calls
|
||||
assert "-v name=ps_restored -v owner=ps_migrate -v loc=en_US.UTF-8" in calls
|
||||
assert "GRANT CONNECT, TEMPORARY" in calls
|
||||
assert re.search(r"^asuser \S*/pg_restore -d ps_restored --exit-on-error --single-transaction$", calls, re.M)
|
||||
assert "packets\t3" in res.stdout
|
||||
assert "DROP DATABASE" not in calls
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_restore_guards(fake: dict[str, object]) -> None:
|
||||
tmp: Path = fake["tmp"] # type: ignore[assignment]
|
||||
dump = tmp / "ps-1.dump"
|
||||
dump.write_text("PGDMP fake")
|
||||
|
||||
res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored", FAKE_DB_EXISTS="1")
|
||||
assert res.returncode == 1
|
||||
assert "exists. Give --replace" in res.stderr
|
||||
|
||||
res = run_restore(fake, "--dump", _posix(dump), "--db", "packetstation", "--replace", FAKE_DB_EXISTS="1")
|
||||
assert res.returncode == 1
|
||||
assert "refused for packetstation" in res.stderr
|
||||
|
||||
res = run_restore(fake, "--dump", _posix(dump), "--db", "Bad-Name")
|
||||
assert res.returncode == 1
|
||||
|
||||
hashed = tmp / "ps-1.globals.sql"
|
||||
hashed.write_text("CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN PASSWORD 'SCRAM-SHA-256$4096:abc';\n")
|
||||
res = run_restore(fake, "--globals", _posix(hashed))
|
||||
assert res.returncode == 1
|
||||
assert "password hashes" in res.stderr
|
||||
|
||||
res = run_restore(fake)
|
||||
assert res.returncode == 1
|
||||
assert "nothing to do" in res.stderr
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_restore_replace_drops_first(fake: dict[str, object]) -> None:
|
||||
dump = fake["tmp"] / "ps-1.dump" # type: ignore[operator]
|
||||
dump.write_text("PGDMP fake")
|
||||
res = run_restore(fake, "--dump", _posix(dump), "--db", "ps_restored", "--replace", FAKE_DB_EXISTS="1")
|
||||
assert res.returncode == 0, res.stderr
|
||||
calls = fake["log"].read_text() # type: ignore[attr-defined]
|
||||
assert calls.index("DROP DATABASE") < calls.index("CREATE DATABASE")
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_restore_globals_skips_existing_roles(fake: dict[str, object]) -> None:
|
||||
g = fake["tmp"] / "ps-1.globals.sql" # type: ignore[operator]
|
||||
g.write_text("CREATE ROLE ps_app;\nALTER ROLE ps_app WITH LOGIN;\n")
|
||||
res = run_restore(fake, "--globals", _posix(g), FAKE_GLOBALS_ERR='ERROR: role "ps_app" already exists')
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert "1 statements skipped" in res.stderr
|
||||
res = run_restore(fake, "--globals", _posix(g), FAKE_GLOBALS_ERR="ERROR: syntax error")
|
||||
assert res.returncode == 1
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_restore_assets_never_overwrite_and_verify(fake: dict[str, object]) -> None:
|
||||
tmp: Path = fake["tmp"] # type: ignore[assignment]
|
||||
src = tmp / "src"
|
||||
(src / "ab").mkdir(parents=True)
|
||||
(src / "ab" / "one").write_bytes(b"one")
|
||||
(src / "ab" / "two").write_bytes(b"two")
|
||||
dst = tmp / "assets"
|
||||
(dst / "ab").mkdir(parents=True)
|
||||
(dst / "ab" / "two").write_bytes(b"kept") # an existing file is never replaced
|
||||
# 0:0 so a run as root (CI containers) has an owner that exists; as a normal user there is no chown.
|
||||
res = run_restore(fake, "--assets", _posix(src), "--assets-dir", _posix(dst), "--assets-owner", "0:0")
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert (dst / "ab" / "one").read_bytes() == b"one"
|
||||
assert (dst / "ab" / "two").read_bytes() == b"kept"
|
||||
|
||||
def rows(files: dict[str, bytes]) -> str:
|
||||
return "".join(f"{hashlib.sha256(v).hexdigest()} {k}\n" for k, v in sorted(files.items()))
|
||||
|
||||
rows_file = tmp / "rows"
|
||||
rows_file.write_text(rows({"ab/one": b"one", "ab/two": b"kept"}), newline="\n")
|
||||
args = ("--verify-assets", "--db", "ps_restored", "--assets-dir", _posix(dst))
|
||||
res = run_restore(fake, *args, FAKE_ASSET_ROWS=_posix(rows_file))
|
||||
assert res.returncode == 0, res.stderr
|
||||
assert "2 rows, 2 match, 0 differ, 0 missing, 0 files with no row" in res.stderr
|
||||
|
||||
(dst / "ab" / "stray").write_bytes(b"x")
|
||||
rows_file.write_text(rows({"ab/one": b"one", "ab/two": b"two", "ab/gone": b"g"}), newline="\n")
|
||||
res = run_restore(fake, *args, FAKE_ASSET_ROWS=_posix(rows_file))
|
||||
assert res.returncode == 1
|
||||
assert "3 rows, 1 match, 1 differ, 1 missing, 1 files with no row" in res.stderr
|
||||
assert "ORPHAN: ab/stray" in res.stderr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- deploy/vm: launcher and ps-run
|
||||
|
||||
|
||||
def _release(tmp: Path) -> Path:
|
||||
rel = tmp / "opt" / "sha-abc1234"
|
||||
(rel / "deploy" / "vm").mkdir(parents=True)
|
||||
(rel / "venv" / "bin").mkdir(parents=True)
|
||||
(rel / "static").mkdir()
|
||||
shutil.copy(VM / "ps-launch.sh", rel / "deploy" / "vm" / "ps-launch.sh")
|
||||
(rel / "deploy" / "vm" / "ps-launch.sh").chmod(0o755)
|
||||
_write_exec(rel / "venv" / "bin" / "packetstation", FAKE_PACKETSTATION)
|
||||
(rel / "BUILD").write_text("sha-abc1234\n", newline="\n")
|
||||
return rel
|
||||
|
||||
|
||||
def _launch(rel: Path, env: dict[str, str], *args: str) -> dict[str, str]:
|
||||
assert BASH is not None
|
||||
base = {k: v for k, v in os.environ.items() if not k.endswith("_FILE") and k != "CREDENTIALS_DIRECTORY"}
|
||||
base.pop("MFILES_PASSWORD", None)
|
||||
base.update(env)
|
||||
res = subprocess.run(
|
||||
[BASH, _posix(rel / "deploy" / "vm" / "ps-launch.sh"), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=base,
|
||||
timeout=60,
|
||||
)
|
||||
assert res.returncode == 0, res.stderr
|
||||
return dict(line.split("=", 1) for line in res.stdout.splitlines())
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_launcher_maps_credentials_and_sets_release_paths(tmp_path: Path) -> None:
|
||||
rel = _release(tmp_path)
|
||||
creds = tmp_path / "creds"
|
||||
creds.mkdir()
|
||||
(creds / "postgres_app_password").write_text("pw-app")
|
||||
(creds / "mfileskit_password").write_text("mf-secret\r\n")
|
||||
(creds / "seal_key").write_text("k" * 32)
|
||||
out = _launch(
|
||||
rel,
|
||||
{
|
||||
"CREDENTIALS_DIRECTORY": _posix(creds),
|
||||
"VISUAL_PASSWORD_FILE": "/etc/packetstation/secrets/visualkit_password",
|
||||
"STATIC_DIR": "/elsewhere",
|
||||
"PS_BUILD": "wrong",
|
||||
},
|
||||
"worker",
|
||||
)
|
||||
c = _posix(creds)
|
||||
assert out["args"] == "worker"
|
||||
assert out["PG_APP_PASSWORD_FILE"] == f"{c}/postgres_app_password"
|
||||
assert out["SEAL_KEY_FILE"] == f"{c}/seal_key"
|
||||
assert out["MFILES_PASSWORD_FILE"] == f"{c}/mfileskit_password"
|
||||
# Not a credential of this unit: the env file's value stays and nothing is invented.
|
||||
assert out["VISUAL_PASSWORD_FILE"] == "/etc/packetstation/secrets/visualkit_password"
|
||||
assert out["PG_MIGRATE_PASSWORD_FILE"] == "<unset>"
|
||||
# Exported from the file, trailing CR LF dropped (the entrypoint drops a CR; the launcher reads with $(<)).
|
||||
assert out["MFILES_PASSWORD"] == "mf-secret"
|
||||
# The release wins over the env file for these.
|
||||
assert out["STATIC_DIR"] == f"{_posix(rel)}/static"
|
||||
assert out["PS_BUILD"] == "sha-abc1234"
|
||||
assert out["WORKER_BEAT_FILE"] == "/run/packetstation/worker.beat"
|
||||
assert out["TZ"] == "UTC"
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_launcher_without_credentials_defaults_to_app(tmp_path: Path) -> None:
|
||||
rel = _release(tmp_path)
|
||||
out = _launch(rel, {"MFILES_PASSWORD_FILE": "/nonexistent/mfileskit_password"})
|
||||
assert out["args"] == "app"
|
||||
assert out["MFILES_PASSWORD"] == "<unset>"
|
||||
assert out["PG_APP_PASSWORD_FILE"] == "<unset>"
|
||||
|
||||
|
||||
@needs_bash
|
||||
def test_ps_run_passes_env_file_credentials_and_sandbox(tmp_path: Path) -> None:
|
||||
rel = _release(tmp_path)
|
||||
secrets = tmp_path / "secrets"
|
||||
secrets.mkdir()
|
||||
(secrets / "postgres_app_password").write_text("x")
|
||||
(secrets / "ldap_bind_password").write_text("y")
|
||||
env_file = tmp_path / "packetstation.env"
|
||||
env_file.write_text("STACK=test\n", newline="\n")
|
||||
fake_run = tmp_path / "systemd-run"
|
||||
_write_exec(fake_run, FAKE_SYSTEMD_RUN)
|
||||
assert BASH is not None
|
||||
env = dict(os.environ)
|
||||
env.update(
|
||||
PS_RUN_ALLOW_NONROOT="1",
|
||||
SYSTEMD_RUN=_posix(fake_run),
|
||||
PS_ENV_FILE_PATH=_posix(env_file),
|
||||
PS_SECRETS_DIR=_posix(secrets),
|
||||
PS_CURRENT=_posix(rel),
|
||||
)
|
||||
res = subprocess.run(
|
||||
[BASH, _posix(VM / "ps-run.sh"), "doctor", "--skip", "visual"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert res.returncode == 0, res.stderr
|
||||
argv = res.stdout.splitlines()
|
||||
s = _posix(secrets)
|
||||
assert argv[:5] == ["--wait", "--pipe", "--collect", "--quiet", "--service-type=exec"]
|
||||
assert "User=packetstation" in argv and "Group=packetstation" in argv
|
||||
assert f"EnvironmentFile={_posix(env_file)}" in argv
|
||||
assert "ProtectSystem=strict" in argv and "NoNewPrivileges=yes" in argv
|
||||
assert f"InaccessiblePaths=-{s}" in argv
|
||||
creds = [a for a in argv if a.startswith("LoadCredential=")]
|
||||
assert creds == [
|
||||
f"LoadCredential=postgres_app_password:{s}/postgres_app_password",
|
||||
f"LoadCredential=ldap_bind_password:{s}/ldap_bind_password",
|
||||
]
|
||||
assert argv[-4:] == [f"{_posix(rel)}/deploy/vm/ps-launch.sh", "doctor", "--skip", "visual"]
|
||||
|
||||
env.pop("PS_RUN_ALLOW_NONROOT")
|
||||
if sys.platform != "win32" and os.geteuid() != 0:
|
||||
res = subprocess.run([BASH, _posix(VM / "ps-run.sh"), "doctor"], capture_output=True, text=True, env=env)
|
||||
assert res.returncode == 2
|
||||
assert "run as root" in res.stderr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- static checks
|
||||
|
||||
|
||||
SCRIPTS = [BACKUP, RESTORE, *sorted(VM.glob("*.sh"))]
|
||||
|
||||
|
||||
@needs_bash
|
||||
@pytest.mark.parametrize("script", SCRIPTS, ids=lambda p: p.name)
|
||||
def test_scripts_are_strict_and_parse(script: Path) -> None:
|
||||
assert BASH is not None
|
||||
for script in (BACKUP, RESTORE):
|
||||
text = script.read_text(encoding="utf-8")
|
||||
assert "\r" not in text
|
||||
assert text.startswith("#!/usr/bin/env bash\n")
|
||||
assert "set -euo pipefail" in text
|
||||
if script in (BACKUP, RESTORE):
|
||||
assert "umask 077" in text
|
||||
if script.name != "make-release.sh":
|
||||
assert "docker exec" not in text and "docker run" not in text
|
||||
res = subprocess.run([BASH, "-n", _posix(script)], capture_output=True, text=True)
|
||||
assert res.returncode == 0, res.stderr
|
||||
|
||||
|
||||
Unit = dict[str, dict[str, list[str]]]
|
||||
|
||||
|
||||
def _unit(name: str) -> Unit:
|
||||
"""A unit file as {section: {key: [values]}}. systemd repeats keys (LoadCredential=, ReadWritePaths=)."""
|
||||
out: Unit = {}
|
||||
section = ""
|
||||
for raw in (UNITS / name).read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
section = line[1:-1]
|
||||
out.setdefault(section, {})
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
out[section].setdefault(key.strip(), []).append(value.strip())
|
||||
return out
|
||||
|
||||
|
||||
def _values(u: Unit, section: str, key: str) -> list[str]:
|
||||
return u.get(section, {}).get(key, [])
|
||||
|
||||
|
||||
LAUNCH = "/opt/packetstation/current/deploy/vm/ps-launch.sh"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unit", "command", "creds"),
|
||||
[
|
||||
("packetstation-migrate.service", "migrate", {"postgres_migrate_password", "postgres_app_password"}),
|
||||
(
|
||||
"packetstation-app.service",
|
||||
"app",
|
||||
{"postgres_app_password", "ldap_bind_password", "session_key", "seal_key"},
|
||||
),
|
||||
(
|
||||
"packetstation-worker.service",
|
||||
"worker",
|
||||
{"postgres_app_password", "visualkit_password", "mfileskit_password", "ldap_bind_password", "seal_key"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_service_units(unit: str, command: str, creds: set[str]) -> None:
|
||||
text = (UNITS / unit).read_text(encoding="utf-8")
|
||||
assert "\r" not in text
|
||||
cp = _unit(unit)
|
||||
svc = "Service"
|
||||
assert _values(cp, svc, "ExecStart") == [f"{LAUNCH} {command}"]
|
||||
assert _values(cp, svc, "User") == ["packetstation"]
|
||||
assert _values(cp, svc, "EnvironmentFile") == ["/etc/packetstation/packetstation.env"]
|
||||
for key, want in (
|
||||
("NoNewPrivileges", "yes"),
|
||||
("ProtectSystem", "strict"),
|
||||
("ProtectHome", "yes"),
|
||||
("PrivateTmp", "yes"),
|
||||
("InaccessiblePaths", "/etc/packetstation/secrets"),
|
||||
("CapabilityBoundingSet", ""),
|
||||
):
|
||||
assert _values(cp, svc, key) == [want], key
|
||||
loaded = {v.split(":", 1)[0] for v in _values(cp, svc, "LoadCredential")}
|
||||
assert loaded == creds
|
||||
for v in _values(cp, svc, "LoadCredential"):
|
||||
name, path = v.split(":", 1)
|
||||
assert path == f"/etc/packetstation/secrets/{name}"
|
||||
rw = _values(cp, svc, "ReadWritePaths")
|
||||
if command == "migrate":
|
||||
assert rw == []
|
||||
else:
|
||||
# The app runs the archive check and the worker copies: both need the archive mount.
|
||||
assert rw == ["/var/lib/packetstation/assets", "-/mnt/packetstation-archive"]
|
||||
assert "packetstation-migrate.service" in _values(cp, "Unit", "Requires")[0]
|
||||
|
||||
|
||||
def test_app_unit_needs_the_firewall_table() -> None:
|
||||
cp = _unit("packetstation-app.service")
|
||||
assert _values(cp, "Service", "ExecStartPre") == ["+/usr/sbin/nft list table inet packetstation"]
|
||||
fw = _unit("packetstation-firewall.service")
|
||||
assert _values(fw, "Service", "ExecStart") == ["/usr/sbin/nft -f /etc/packetstation/packetstation.nft"]
|
||||
|
||||
|
||||
def test_launcher_knows_every_credential_the_units_load() -> None:
|
||||
launch = (VM / "ps-launch.sh").read_text(encoding="utf-8")
|
||||
run = (VM / "ps-run.sh").read_text(encoding="utf-8")
|
||||
names: set[str] = set()
|
||||
for unit in UNITS.glob("*.service"):
|
||||
names |= {v.split(":", 1)[0] for v in _values(_unit(unit.name), "Service", "LoadCredential")}
|
||||
for n in names:
|
||||
assert re.search(rf"(^|')({n}) [A-Z_]+_FILE'?$", launch, re.M), n
|
||||
assert n in run, n
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stack", ["prod", "test"])
|
||||
def test_env_examples(stack: str) -> None:
|
||||
text = (VM / f"{stack}.env.example").read_text(encoding="utf-8")
|
||||
assert "\r" not in text
|
||||
values = dict(line.split("=", 1) for line in text.splitlines() if line and not line.startswith("#") and "=" in line)
|
||||
assert values["STACK"] == stack
|
||||
assert values["PG_HOST"] == "/var/run/postgresql"
|
||||
assert values["TRUSTED_PROXY_HOST"] == "192.168.101.12"
|
||||
assert values["ASSETS_DIR"] == "/var/lib/packetstation/assets"
|
||||
assert values["BACKUP_DIR"] == "/var/backups/packetstation"
|
||||
assert "#ARCHIVE_ROOT=/mnt/packetstation-archive" in text
|
||||
for k, v in values.items():
|
||||
if k.endswith("_FILE"):
|
||||
assert v.startswith("/etc/packetstation/secrets/"), k
|
||||
assert "PASSWORD=" not in f"{k}=", k
|
||||
for k in ("STATIC_DIR", "PS_BUILD", "WORKER_BEAT_FILE", "MFILES_CA_BUNDLE"):
|
||||
assert k not in values, k
|
||||
if stack == "prod":
|
||||
assert (values["VISUAL_SERVER"], values["VISUAL_DATABASE"]) == ("vsql.pesco.com", "PESCOINC")
|
||||
assert values["NOTIFY_DRY_RUN"] == "false"
|
||||
else:
|
||||
assert (values["VISUAL_SERVER"], values["VISUAL_DATABASE"]) == ("'vsql.pesco.com\\DEV'", "PESCOINCDEV")
|
||||
assert values["NOTIFY_DRY_RUN"] == "true"
|
||||
|
||||
|
||||
def test_backup_units() -> None:
|
||||
svc = (REPO / "scripts" / "backup" / "packetstation-backup.service").read_text(encoding="utf-8")
|
||||
timer = (REPO / "scripts" / "backup" / "packetstation-backup.timer").read_text(encoding="utf-8")
|
||||
assert "ExecStart=/usr/local/sbin/ps-backup --config /etc/packetstation/backup.conf" in svc
|
||||
assert "docker" not in svc.lower()
|
||||
assert "OnCalendar=*-*-* 02:00:00 America/Denver" in timer
|
||||
assert "Persistent=true" in timer
|
||||
|
||||
Reference in New Issue
Block a user