281 lines
11 KiB
Bash
Executable File
281 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Install a Packet Station release on this VM and switch to it. Run by Spencer as root.
|
|
# Installed as /usr/local/sbin/ps-upgrade by deploy/vm/install.sh.
|
|
#
|
|
# Usage:
|
|
# ps-upgrade RELEASE.tar.gz [--start] [--no-backup] [--no-doctor]
|
|
# ps-upgrade --rollback
|
|
#
|
|
# RELEASE.tar.gz comes from deploy/vm/make-release.sh. When RELEASE.tar.gz.sha256 lies next to it,
|
|
# the archive is checked against it first.
|
|
#
|
|
# Steps:
|
|
# 1. Unpack into /opt/packetstation/<build> and build its venv there: public pins from PyPI
|
|
# (server/requirements.txt), visualkit and mfileskit from the Gitea index with --index-url
|
|
# (server/requirements-pesco.txt), then the package itself with --no-deps; pip check.
|
|
# A release folder that is already complete is reused.
|
|
# 2. A backup (ps-backup) when the database exists, unless --no-backup.
|
|
# 3. Stop the app and the worker, point /opt/packetstation/current at the new release
|
|
# (previous keeps the old one), install its units and commands (install.sh --files-only).
|
|
# 4. Run the migrations (restart packetstation-migrate.service). On failure: current goes back,
|
|
# the services stay stopped, and the restore command is printed.
|
|
# 5. Start again whichever of the app and the worker ran before. A first install starts nothing
|
|
# unless --start: on production the first start is Spencer's approval of the VISUAL polling.
|
|
# 6. Wait for /healthz, run ps-doctor (unless --no-doctor) and keep the newest 3 releases.
|
|
#
|
|
# --rollback points current back at previous and restarts what ran. Migrations are not undone:
|
|
# roll back only to a release whose code works with the schema now in the database.
|
|
#
|
|
# Python: PYTHON (default python3.12) must be 3.12 or newer.
|
|
set -euo pipefail
|
|
umask 022
|
|
|
|
OPT="${PS_OPT:-/opt/packetstation}"
|
|
PYTHON="${PYTHON:-python3.12}"
|
|
PESCO_INDEX_URL="${PESCO_INDEX_URL:-https://gt.pescoinc.biz/api/packages/spencerm/pypi/simple/}"
|
|
PYPI_INDEX_URL="${PYPI_INDEX_URL:-https://pypi.org/simple/}"
|
|
KEEP_RELEASES="${KEEP_RELEASES:-3}"
|
|
HEALTH_URL="http://127.0.0.1:8000/healthz"
|
|
SERVICES=(packetstation-app.service packetstation-worker.service)
|
|
|
|
log() { printf 'ps-upgrade: %s\n' "$*" >&2; }
|
|
die() { printf 'ps-upgrade: ERROR: %s\n' "$*" >&2; exit 1; }
|
|
usage() { sed -n '2,31p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
|
|
|
tarball=""
|
|
start=0
|
|
backup=1
|
|
doctor=1
|
|
rollback=0
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--start) start=1; shift ;;
|
|
--no-backup) backup=0; shift ;;
|
|
--no-doctor) doctor=0; shift ;;
|
|
--rollback) rollback=1; shift ;;
|
|
-h|--help) usage ;;
|
|
-*) log "unknown option $1"; usage ;;
|
|
*) [[ -z "$tarball" ]] || usage; tarball="$1"; shift ;;
|
|
esac
|
|
done
|
|
|
|
[[ "$(id -u)" == "0" ]] || die "run as root"
|
|
[[ $rollback -eq 1 || -n "$tarball" ]] || usage
|
|
[[ "$KEEP_RELEASES" =~ ^[1-9][0-9]*$ ]] || die "KEEP_RELEASES must be a positive whole number"
|
|
|
|
exec 9>/run/lock/packetstation-upgrade.lock
|
|
flock -n 9 || die "another ps-upgrade is running"
|
|
|
|
# ------------------------------------------------------------------ helpers
|
|
|
|
was_active=()
|
|
remember_active() {
|
|
local s
|
|
was_active=()
|
|
for s in "${SERVICES[@]}"; do
|
|
if systemctl is-active --quiet "$s"; then was_active+=("$s"); fi
|
|
done
|
|
}
|
|
|
|
switch_to() {
|
|
# Atomic: a new link next to current, then rename over it.
|
|
local target="$1"
|
|
ln -sfn -- "$target" "$OPT/.current.new"
|
|
mv -T -- "$OPT/.current.new" "$OPT/current"
|
|
}
|
|
|
|
wait_healthy() {
|
|
local i
|
|
for (( i = 0; i < 30; i++ )); do
|
|
if curl -fsS --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
|
|
log "the app answers $HEALTH_URL"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
log "WARNING: the app did not answer $HEALTH_URL within 60 s: journalctl -u packetstation-app -n 50"
|
|
return 1
|
|
}
|
|
|
|
start_again() {
|
|
local -a units=("$@")
|
|
if [[ ${#units[@]} -eq 0 ]]; then
|
|
log "the app and the worker were not running and stay stopped."
|
|
log "start them with: systemctl enable --now packetstation-app packetstation-worker"
|
|
return 0
|
|
fi
|
|
log "starting ${units[*]}"
|
|
systemctl start "${units[@]}"
|
|
local s
|
|
for s in "${units[@]}"; do
|
|
if [[ "$s" == packetstation-app.service ]]; then wait_healthy || true; fi
|
|
done
|
|
}
|
|
|
|
prune() {
|
|
local cur prev d
|
|
cur="$(readlink -f -- "$OPT/current" 2>/dev/null || true)"
|
|
prev="$(readlink -f -- "$OPT/previous" 2>/dev/null || true)"
|
|
local -a dirs=()
|
|
while IFS= read -r d; do dirs+=("$d"); done < <(
|
|
find "$OPT" -mindepth 1 -maxdepth 1 -type d -name '[A-Za-z0-9]*' -printf '%T@ %p\n' | sort -rn | cut -d' ' -f2-
|
|
)
|
|
# current and previous always stay; they count toward KEEP_RELEASES.
|
|
local allowed=$(( KEEP_RELEASES - 2 )) kept=0
|
|
for d in "${dirs[@]}"; do
|
|
if [[ "$d" == "$cur" || "$d" == "$prev" ]]; then continue; fi
|
|
kept=$(( kept + 1 ))
|
|
if [[ $kept -gt $allowed ]]; then
|
|
log "removing old release $(basename -- "$d")"
|
|
rm -rf -- "$d"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ------------------------------------------------------------------ rollback
|
|
|
|
if [[ $rollback -eq 1 ]]; then
|
|
[[ -L "$OPT/previous" ]] || die "no previous release to go back to"
|
|
prev="$(readlink -f -- "$OPT/previous")"
|
|
cur="$(readlink -f -- "$OPT/current")"
|
|
[[ -f "$prev/.ready" ]] || die "previous release $prev is not complete"
|
|
remember_active
|
|
log "rolling back from $(basename -- "$cur") to $(basename -- "$prev")"
|
|
if [[ ${#was_active[@]} -gt 0 ]]; then systemctl stop "${was_active[@]}"; fi
|
|
switch_to "$prev"
|
|
ln -sfn -- "$cur" "$OPT/previous"
|
|
bash "$prev/deploy/vm/install.sh" --files-only
|
|
start_again "${was_active[@]}"
|
|
log "done. Migrations were not undone."
|
|
exit 0
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ 1. unpack and build
|
|
|
|
[[ -r "$tarball" ]] || die "cannot read $tarball"
|
|
if [[ -r "$tarball.sha256" ]]; then
|
|
( cd -- "$(dirname -- "$tarball")" && sha256sum --quiet -c -- "$(basename -- "$tarball").sha256" ) \
|
|
|| die "$tarball does not match $tarball.sha256"
|
|
log "checksum ok"
|
|
else
|
|
log "no $tarball.sha256 next to the archive; checksum not checked"
|
|
fi
|
|
|
|
# Every member must sit under one top folder packetstation-<build>/, with no absolute path and no "..".
|
|
# The listing is read whole first: an early-exiting reader (head, grep -q) would make tar fail
|
|
# with SIGPIPE under pipefail.
|
|
members="$(tar -tzf "$tarball")" || die "cannot list $tarball"
|
|
top="${members%%$'\n'*}"
|
|
top="${top%%/*}"
|
|
[[ "$top" =~ ^packetstation-([A-Za-z0-9][A-Za-z0-9._-]{0,63})$ ]] || die "unexpected top folder '$top' in $tarball"
|
|
build="${BASH_REMATCH[1]}"
|
|
outside="$(grep -Evc "^$top(/|$)" <<< "$members" || true)"
|
|
[[ "$outside" == "0" ]] || die "$tarball has files outside $top/"
|
|
dotdot="$(grep -Ec '(^|/)\.\.(/|$)' <<< "$members" || true)"
|
|
[[ "$dotdot" == "0" ]] || die "$tarball has a '..' path"
|
|
unset members
|
|
|
|
dest="$OPT/$build"
|
|
if [[ -f "$dest/.ready" ]]; then
|
|
log "release $build is already built in $dest"
|
|
else
|
|
"$PYTHON" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)' \
|
|
|| die "$PYTHON is not Python 3.12 or newer (set PYTHON=/path/to/python3.12)"
|
|
rm -rf -- "$dest"
|
|
install -d -o root -g root -m 0755 "$dest"
|
|
log "unpacking $build into $dest"
|
|
tar -xzf "$tarball" -C "$dest" --strip-components=1 --no-same-owner --no-same-permissions
|
|
[[ "$(head -n 1 "$dest/BUILD")" == "$build" ]] || die "BUILD in the archive does not say $build"
|
|
[[ -f "$dest/static/index.html" ]] || die "the archive has no built frontend (static/index.html)"
|
|
|
|
log "building the venv"
|
|
"$PYTHON" -m venv "$dest/venv"
|
|
pip=("$dest/venv/bin/python" -m pip --disable-pip-version-check --no-input)
|
|
# No pip.conf or PIP_* setting from the host may add an index.
|
|
export PIP_CONFIG_FILE=/dev/null
|
|
unset PIP_INDEX_URL PIP_EXTRA_INDEX_URL PIP_FIND_LINKS
|
|
"${pip[@]}" install --index-url "$PYPI_INDEX_URL" -r "$dest/server/requirements.txt"
|
|
# PESCO libraries from the Gitea index only. --index-url, never --extra-index-url.
|
|
"${pip[@]}" install --index-url "$PESCO_INDEX_URL" -r "$dest/server/requirements-pesco.txt"
|
|
# The package itself. The build backend (hatchling) comes from PyPI.
|
|
"${pip[@]}" install --index-url "$PYPI_INDEX_URL" --no-deps "$dest"
|
|
"${pip[@]}" check
|
|
# The release is read-only to the services. Owned by root, nothing writable by others.
|
|
chown -R root:root "$dest"
|
|
chmod -R u+rwX,go+rX,go-w "$dest"
|
|
# The units and ps-run start these directly; an archive made on Windows may have lost the x bit.
|
|
chmod 0755 "$dest"/deploy/vm/*.sh "$dest"/scripts/backup/*.sh
|
|
printf '%s\n' "$build" > "$dest/.ready"
|
|
log "release $build built"
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ 2. backup
|
|
|
|
first_install=1
|
|
[[ -L "$OPT/current" ]] && first_install=0
|
|
|
|
db_exists=0
|
|
found="$(setpriv --reuid=postgres --regid=postgres --init-groups -- \
|
|
psql -X -At -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'packetstation'" 2>/dev/null || true)"
|
|
if [[ "$found" == "1" ]]; then
|
|
db_exists=1
|
|
fi
|
|
if [[ $backup -eq 1 && $db_exists -eq 1 && $first_install -eq 0 ]]; then
|
|
log "backup before the upgrade"
|
|
/usr/local/sbin/ps-backup || die "the backup failed; nothing was changed. Fix it, or use --no-backup."
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ 3. switch
|
|
|
|
remember_active
|
|
if [[ $start -eq 1 ]]; then was_active=("${SERVICES[@]}"); fi
|
|
old=""
|
|
if [[ $first_install -eq 0 ]]; then
|
|
old="$(readlink -f -- "$OPT/current")"
|
|
fi
|
|
if [[ "$old" == "$dest" ]]; then
|
|
log "release $build is already current"
|
|
fi
|
|
|
|
active_now=()
|
|
for s in "${SERVICES[@]}"; do
|
|
if systemctl is-active --quiet "$s"; then active_now+=("$s"); fi
|
|
done
|
|
if [[ ${#active_now[@]} -gt 0 ]]; then
|
|
log "stopping ${active_now[*]}"
|
|
systemctl stop "${active_now[@]}"
|
|
fi
|
|
|
|
switch_to "$dest"
|
|
if [[ -n "$old" && "$old" != "$dest" ]]; then
|
|
ln -sfn -- "$old" "$OPT/previous"
|
|
fi
|
|
log "current is $build"
|
|
bash "$dest/deploy/vm/install.sh" --files-only
|
|
|
|
# ------------------------------------------------------------------ 4. migrate
|
|
|
|
log "running the migrations"
|
|
if ! systemctl restart packetstation-migrate.service; then
|
|
journalctl -u packetstation-migrate.service -n 30 --no-pager >&2 || true
|
|
if [[ -n "$old" && "$old" != "$dest" ]]; then
|
|
switch_to "$old"
|
|
bash "$old/deploy/vm/install.sh" --files-only || true
|
|
log "current is back on $(basename -- "$old")"
|
|
fi
|
|
last="$(find /var/backups/packetstation -maxdepth 1 -name 'ps-*.dump' -printf '%f\n' 2>/dev/null | sort | tail -n 1)"
|
|
die "the migrations failed. The app and the worker are stopped. Read the log above. If the database
|
|
must go back to the backup taken before this upgrade, restore it (docs/DEPLOY.md, \"Restore on a VM\"):
|
|
ps-restore --dump /var/backups/packetstation/${last:-<dump>} --db ps_restored"
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ 5. and 6. start, check, prune
|
|
|
|
start_again "${was_active[@]}"
|
|
if [[ $doctor -eq 1 ]]; then
|
|
log "doctor"
|
|
/usr/local/sbin/ps-doctor || log "WARNING: the doctor reported a failure (see above)"
|
|
fi
|
|
prune
|
|
log "done: $build"
|