Native VM install kit: systemd units, launcher, install and upgrade scripts; backups, restore and doctor without docker exec; DEPLOY.md VM, NPM and NFS sections
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
+44
-60
@@ -1,50 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# Packet Station nightly backup. Runs on the docker host, never inside the app.
|
||||
# Packet Station nightly backup on a native VM. Runs as root from packetstation-backup.timer,
|
||||
# outside the app, so a broken app still backs up.
|
||||
#
|
||||
# For each configured stack it writes, into that stack's backup folder:
|
||||
# ps-<UTC stamp>.dump pg_dump -Fc of the app database (custom format)
|
||||
# ps-<UTC stamp>.globals.sql pg_dumpall --globals-only --no-role-passwords (roles, no password hashes)
|
||||
# last-success / last-failure one line: "<ISO UTC time> <dump name>" (last-failure adds the reason)
|
||||
#
|
||||
# Each dump is written under a temporary name, checked with `pg_restore --list` run inside the
|
||||
# postgres container, and only then renamed into place. The newest KEEP dumps (default 7) are kept,
|
||||
# with their globals files. Any failure writes last-failure and the script exits non-zero after
|
||||
# trying the remaining stacks.
|
||||
# Each dump is written under a temporary name, checked with `pg_restore --list`, and only then
|
||||
# renamed into place. The newest KEEP dumps (default 7) are kept, with their globals files. Any
|
||||
# failure writes last-failure and the script exits non-zero after trying the remaining stacks.
|
||||
#
|
||||
# Usage:
|
||||
# ps-backup.sh [--config FILE] [STACK ...]
|
||||
# ps-backup.sh --stack NAME CONTAINER DATABASE FOLDER [--stack ...]
|
||||
# ps-backup [--config FILE] [--keep N] [STACK ...]
|
||||
# ps-backup --stack NAME DATABASE FOLDER [--stack ...] [--keep N]
|
||||
#
|
||||
# Config file (default /etc/packetstation/backup.conf, or PS_BACKUP_CONFIG), one stack per line:
|
||||
# # name container database folder
|
||||
# prod project:packetstation packetstation /var/backups/packetstation/prod
|
||||
# test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||||
# CONTAINER is a container name, or project:<compose project> to find that project's postgres
|
||||
# service by its compose labels. With STACK names given, only those stacks run.
|
||||
# # name database folder
|
||||
# prod packetstation /var/backups/packetstation
|
||||
# With STACK names given, only those stacks run. A VM has one stack; install.sh writes its line.
|
||||
#
|
||||
# pg_dump connects inside the container as the postgres superuser over the local socket, which
|
||||
# the official image trusts. No password is read, passed or printed.
|
||||
# pg_dump and pg_dumpall run as the postgres OS user (setpriv, no PAM) on the local socket, where
|
||||
# PostgreSQL knows that user by peer authentication. No password is read, passed or printed. Their
|
||||
# output goes to files root opened, so the postgres user needs no access to the folder.
|
||||
# PG_BINDIR (default /usr/lib/postgresql/16/bin) picks the client tools, PG_AS the user switch.
|
||||
#
|
||||
# Globals: --no-role-passwords, so no password hash ever lands in a file (or in the PBS copy of
|
||||
# it). The roles' passwords come from the stack's secrets: initdb (deploy/postgres/initdb)
|
||||
# recreates them on a fresh volume. The globals file is also 0600, like the dumps.
|
||||
# it). The role passwords are in /etc/packetstation/secrets; deploy/vm/install.sh sets them.
|
||||
#
|
||||
# Files: dumps and globals are 0600. The markers are 0644 (a time and a file name, nothing else)
|
||||
# so the worker, which mounts the folder read-only as uid 10001, can read them. When run as root
|
||||
# the new files take the folder's owner and group, so a stack folder owned by spencerm stays
|
||||
# readable by spencerm.
|
||||
# so the worker, which reaches the folder through its group, can read them. When run as root the
|
||||
# new files take the folder's owner and group.
|
||||
#
|
||||
# Not in the dump: the stored files (/var/lib/packetstation/assets), the env file and the secrets
|
||||
# (/etc/packetstation). The PBS backup of the VM covers them. See docs/DEPLOY.md.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
KEEP="${KEEP:-7}"
|
||||
PG_USER="${PG_USER:-postgres}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}"
|
||||
# The command prefix that runs a client tool as the postgres OS user, split on spaces.
|
||||
read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}"
|
||||
|
||||
log() { printf '%s ps-backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||||
sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
@@ -53,7 +55,7 @@ usage() {
|
||||
config="${PS_BACKUP_CONFIG:-/etc/packetstation/backup.conf}"
|
||||
config_given=0
|
||||
declare -a only=()
|
||||
declare -a stacks=() # each entry: "name|container|database|folder"
|
||||
declare -a stacks=() # each entry: "name|database|folder"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -61,8 +63,8 @@ while [[ $# -gt 0 ]]; do
|
||||
[[ $# -ge 2 ]] || usage
|
||||
config="$2"; config_given=1; shift 2 ;;
|
||||
--stack)
|
||||
[[ $# -ge 5 ]] || usage
|
||||
stacks+=("$2|$3|$4|$5"); shift 5 ;;
|
||||
[[ $# -ge 4 ]] || usage
|
||||
stacks+=("$2|$3|$4"); shift 4 ;;
|
||||
--keep)
|
||||
[[ $# -ge 2 ]] || usage
|
||||
KEEP="$2"; shift 2 ;;
|
||||
@@ -82,13 +84,13 @@ if [[ ${#stacks[@]} -eq 0 ]]; then
|
||||
log "no --stack given and the config $config is missing or unreadable"
|
||||
exit 2
|
||||
fi
|
||||
while read -r name container database folder extra; do
|
||||
while read -r name database folder extra; do
|
||||
[[ -z "${name:-}" || "${name:0:1}" == "#" ]] && continue
|
||||
if [[ -z "${folder:-}" || -n "${extra:-}" ]]; then
|
||||
log "config line for '$name' needs exactly: name container database folder"
|
||||
log "config line for '$name' needs exactly: name database folder"
|
||||
exit 2
|
||||
fi
|
||||
stacks+=("$name|$container|$database|$folder")
|
||||
stacks+=("$name|$database|$folder")
|
||||
done < "$config"
|
||||
elif [[ $config_given -eq 1 ]]; then
|
||||
log "use --config or --stack, not both"
|
||||
@@ -102,26 +104,9 @@ fi
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
# Resolve "project:<name>" to the running postgres container of that compose project.
|
||||
resolve_container() {
|
||||
local spec="$1"
|
||||
if [[ "$spec" == project:* ]]; then
|
||||
local project="${spec#project:}" ids
|
||||
ids="$("$DOCKER" ps -q \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=postgres")"
|
||||
if [[ -z "$ids" ]]; then
|
||||
echo "no running postgres container for compose project $project" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(printf '%s\n' "$ids" | wc -l)" -ne 1 ]]; then
|
||||
echo "more than one postgres container for compose project $project" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$ids"
|
||||
else
|
||||
printf '%s\n' "$spec"
|
||||
fi
|
||||
# A PostgreSQL client tool: from PG_BINDIR when it is set, else from PATH.
|
||||
pgbin() {
|
||||
if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi
|
||||
}
|
||||
|
||||
# Write a marker atomically: temp file in the same folder, then rename.
|
||||
@@ -203,8 +188,8 @@ on_stack_exit() {
|
||||
}
|
||||
|
||||
backup_stack() {
|
||||
local name="$1" spec="$2" database="$3" folder="$4"
|
||||
local stamp globals_name container started entries size
|
||||
local name="$1" database="$2" folder="$3"
|
||||
local stamp globals_name started entries size
|
||||
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
cur_dump_name="ps-$stamp.dump"
|
||||
@@ -213,34 +198,33 @@ backup_stack() {
|
||||
log "stack $name: database $database into $folder"
|
||||
|
||||
if [[ ! -d "$folder" ]]; then
|
||||
# 0750: the worker reaches the folder through its group (see scripts/backup/README.md).
|
||||
# 0750: the worker reaches the folder through its group (docs/DEPLOY.md, "Backups on a VM").
|
||||
mkdir -p -- "$folder"
|
||||
chmod 0750 -- "$folder" 2>/dev/null || log "stack $name: could not set mode 0750 on $folder"
|
||||
fi
|
||||
cur_folder="$folder"
|
||||
[[ -w "$folder" ]] || fail_stack "$folder is not writable"
|
||||
[[ "$database" =~ ^[A-Za-z_][A-Za-z0-9_]{0,62}$ ]] || fail_stack "database name '$database' is not a plain name"
|
||||
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
exec 9>"$folder/.ps-backup.lock"
|
||||
flock -n 9 || fail_stack "another backup of this stack is running"
|
||||
fi
|
||||
|
||||
container="$(resolve_container "$spec" 2>&1)" || fail_stack "$container"
|
||||
|
||||
cur_tmp_dump="$(mktemp "$folder/.$cur_dump_name.XXXXXX.partial")"
|
||||
cur_tmp_globals="$(mktemp "$folder/.$globals_name.XXXXXX.partial")"
|
||||
|
||||
"$DOCKER" exec "$container" pg_dump -U "$PG_USER" -d "$database" -Fc > "$cur_tmp_dump" \
|
||||
"${PG_AS[@]}" "$(pgbin pg_dump)" -d "$database" -Fc > "$cur_tmp_dump" \
|
||||
|| fail_stack "pg_dump failed"
|
||||
[[ -s "$cur_tmp_dump" ]] || fail_stack "pg_dump wrote nothing"
|
||||
|
||||
"$DOCKER" exec "$container" pg_dumpall -U "$PG_USER" --globals-only --no-role-passwords \
|
||||
"${PG_AS[@]}" "$(pgbin pg_dumpall)" --globals-only --no-role-passwords \
|
||||
> "$cur_tmp_globals" || fail_stack "pg_dumpall --globals-only failed"
|
||||
[[ -s "$cur_tmp_globals" ]] || fail_stack "pg_dumpall wrote nothing"
|
||||
|
||||
# The check reads the dump from stdin with the container's own pg_restore. Entry lines start
|
||||
# with the entry's id; comment lines start with a semicolon.
|
||||
entries="$("$DOCKER" exec -i "$container" pg_restore --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
|
||||
# The check reads the dump from stdin and needs no database. Entry lines start with the
|
||||
# entry's id; comment lines start with a semicolon.
|
||||
entries="$("$(pgbin pg_restore)" --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
|
||||
|| fail_stack "pg_restore --list could not read the dump"
|
||||
[[ "$entries" -ge 1 ]] || fail_stack "pg_restore --list found no entries"
|
||||
|
||||
@@ -269,7 +253,7 @@ run_stack() (
|
||||
failures=0
|
||||
ran=0
|
||||
for entry in "${stacks[@]}"; do
|
||||
IFS='|' read -r s_name s_container s_database s_folder <<< "$entry"
|
||||
IFS='|' read -r s_name s_database s_folder <<< "$entry"
|
||||
if [[ ${#only[@]} -gt 0 ]]; then
|
||||
wanted=0
|
||||
for o in "${only[@]}"; do
|
||||
@@ -280,7 +264,7 @@ for entry in "${stacks[@]}"; do
|
||||
ran=$(( ran + 1 ))
|
||||
# Not in an if or ||: that would switch errexit off inside the subshell.
|
||||
set +e
|
||||
run_stack "$s_name" "$s_container" "$s_database" "$s_folder"
|
||||
run_stack "$s_name" "$s_database" "$s_folder"
|
||||
rc=$?
|
||||
set -e
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
|
||||
Reference in New Issue
Block a user