#!/usr/bin/env bash # Packet Station restore on a native VM. Run by root. Restores a ps-backup dump into a fresh # database, optionally applies the globals file, copies stored files back into an assets folder, # checks them against the database, and prints the row count of every table at the end. # # It never stops, starts or reconfigures a service. Putting the restored database into service # (renaming it into place) is a separate, deliberate step: see docs/DEPLOY.md, "Restore on a VM". # # Usage: # ps-restore [--dump FILE --db NAME [--replace]] # [--globals FILE] # [--assets DIR_OR_TAR --assets-dir DIR] # [--verify-assets --db NAME --assets-dir DIR] # [--counts-only --db NAME] # # --dump FILE a ps-*.dump file (pg_dump -Fc). # --db NAME the database to restore into. It must not exist, unless --replace, # which drops it first. DROP DATABASE refuses while anyone is connected. # --replace is refused for packetstation, the live database. # --owner ROLE owner of the new database (default ps_migrate, as install.sh makes it). # --app-role ROLE role given CONNECT and TEMPORARY on it (default ps_app). # --locale NAME the new database's locale (default en_US.UTF-8, as install.sh makes it). # --globals FILE a ps-*.globals.sql file. Roles that already exist are reported and # left alone. The file holds no passwords: install.sh sets them again. # --assets SRC a folder, or a .tar / .tar.gz / .tgz, holding the files that belong # under ASSETS_DIR. Existing files are never overwritten. # --assets-dir DIR the folder to copy into or check (/var/lib/packetstation/assets is the # live one; a scratch folder for a rehearsal). # --assets-owner U:G owner for the copied files (default packetstation:packetstation). # --verify-assets compare sha256 of every file in --assets-dir with assets.sha256 in --db. # --counts-only only print the row counts of --db. # # psql and pg_restore run as the postgres OS user on the local socket (peer authentication). The # files are opened by root and handed over on stdin. No password is read, passed or printed. set -euo pipefail umask 077 PG_BINDIR="${PG_BINDIR-/usr/lib/postgresql/16/bin}" read -r -a PG_AS <<< "${PG_AS:-setpriv --reuid=postgres --regid=postgres --init-groups --reset-env --}" LIVE_DB=packetstation log() { printf '%s ps-restore: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; } die() { log "$*"; exit 1; } usage() { sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } dump="" db="" replace=0 owner="ps_migrate" app_role="ps_app" locale_name="en_US.UTF-8" globals="" assets="" assets_dir="" assets_owner="packetstation:packetstation" verify_assets=0 counts_only=0 while [[ $# -gt 0 ]]; do case "$1" in --dump) dump="${2:?}"; shift 2 ;; --db) db="${2:?}"; shift 2 ;; --replace) replace=1; shift ;; --owner) owner="${2:?}"; shift 2 ;; --app-role) app_role="${2:?}"; shift 2 ;; --locale) locale_name="${2:?}"; shift 2 ;; --globals) globals="${2:?}"; shift 2 ;; --assets) assets="${2:?}"; shift 2 ;; --assets-dir) assets_dir="${2:?}"; shift 2 ;; --assets-owner) assets_owner="${2:?}"; shift 2 ;; --verify-assets) verify_assets=1; shift ;; --counts-only) counts_only=1; shift ;; -h|--help) usage ;; *) log "unknown argument $1"; usage ;; esac done ident_re='^[a-z_][a-z0-9_]{0,62}$' [[ -z "$db" || "$db" =~ $ident_re ]] || die "--db must be a plain lower-case name (letters, digits, _)" [[ "$owner" =~ $ident_re ]] || die "--owner must be a plain lower-case role name" [[ "$app_role" =~ $ident_re ]] || die "--app-role must be a plain lower-case role name" [[ "$locale_name" =~ ^[A-Za-z0-9_.@-]+$ ]] || die "--locale must be a locale name like en_US.UTF-8" [[ "$assets_owner" =~ ^[a-z_0-9][a-z0-9_-]*:[a-z_0-9][a-z0-9_-]*$ ]] || die "--assets-owner must be USER:GROUP" if [[ -n "$dump" || $counts_only -eq 1 || $verify_assets -eq 1 ]]; then [[ -n "$db" ]] || die "--db is required" fi if [[ -n "$assets" || $verify_assets -eq 1 ]]; then [[ -n "$assets_dir" ]] || die "--assets-dir is required" [[ -d "$assets_dir" ]] || die "$assets_dir is not a folder" fi if [[ -z "$dump" && -z "$globals" && -z "$assets" && $verify_assets -eq 0 && $counts_only -eq 0 ]]; then die "nothing to do: give --dump, --globals, --assets, --verify-assets or --counts-only" fi if [[ -n "$dump" && $replace -eq 1 && "$db" == "$LIVE_DB" ]]; then die "--replace is refused for $LIVE_DB. Restore into another name and rename it into place (docs/DEPLOY.md)." fi pgbin() { if [[ -n "$PG_BINDIR" ]]; then printf '%s/%s\n' "$PG_BINDIR" "$1"; else printf '%s\n' "$1"; fi } # psql as postgres, reading SQL from stdin. Variables go through -v and :'name'. psql_c() { local dbname="$1"; shift "${PG_AS[@]}" "$(pgbin psql)" -X -q -v ON_ERROR_STOP=1 -d "$dbname" "$@" } db_exists() { local n n="$(psql_c postgres -At -v name="$1" <<'SQL' SELECT count(*) FROM pg_database WHERE datname = :'name'; SQL )" [[ "$n" == "1" ]] } role_exists() { local n n="$(psql_c postgres -At -v name="$1" <<'SQL' SELECT count(*) FROM pg_roles WHERE rolname = :'name'; SQL )" [[ "$n" == "1" ]] } print_counts() { local dbname="$1" log "row counts in $dbname" psql_c "$dbname" -A -F $'\t' -P footer=off <<'SQL' SELECT c.relname AS table_name, (xpath('/row/n/text()', query_to_xml(format('SELECT count(*) AS n FROM %I.%I', n.nspname, c.relname), false, true, '') ))[1]::text::bigint AS row_count FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') ORDER BY c.relname; SQL } # ------------------------------------------------------------------ globals if [[ -n "$globals" ]]; then [[ -r "$globals" ]] || die "cannot read $globals" if grep -qiE "PASSWORD[[:space:]]+'(SCRAM|md5)" "$globals"; then die "$globals holds password hashes; ps-backup writes globals without them. Refusing." fi log "applying globals from $(basename -- "$globals") (existing roles are left as they are)" # Without ON_ERROR_STOP: CREATE ROLE for a role that exists fails and the rest goes on. out="$("${PG_AS[@]}" "$(pgbin psql)" -X -q -d postgres < "$globals" 2>&1 || true)" exists="$(printf '%s\n' "$out" | grep -c 'already exists' || true)" other="$(printf '%s\n' "$out" | grep 'ERROR' | grep -vc 'already exists' || true)" log "globals: $exists statements skipped (already exists), $other other errors" if [[ "$other" -gt 0 ]]; then printf '%s\n' "$out" | grep 'ERROR' | grep -v 'already exists' >&2 die "globals had errors" fi fi # ------------------------------------------------------------------ database if [[ -n "$dump" ]]; then [[ -r "$dump" ]] || die "cannot read $dump" started=$SECONDS if db_exists "$db"; then [[ $replace -eq 1 ]] || die "database $db exists. Give --replace to drop it first." log "dropping database $db (--replace)" psql_c postgres -v name="$db" <<'SQL' SELECT format('DROP DATABASE %I', :'name') \gexec SQL fi role_exists "$owner" || die "role $owner does not exist (apply --globals, or run deploy/vm/install.sh)" log "creating database $db owned by $owner" # The same shape as deploy/vm/install.sh makes the live one. psql_c postgres -v name="$db" -v owner="$owner" -v loc="$locale_name" <<'SQL' SELECT format('CREATE DATABASE %I OWNER %I ENCODING %L LOCALE %L TEMPLATE template0', :'name', :'owner', 'UTF8', :'loc') \gexec SELECT format('REVOKE ALL ON DATABASE %I FROM PUBLIC', :'name') \gexec SQL if role_exists "$app_role"; then psql_c postgres -v name="$db" -v app="$app_role" <<'SQL' SELECT format('GRANT CONNECT, TEMPORARY ON DATABASE %I TO %I', :'name', :'app') \gexec SQL else log "role $app_role does not exist; no CONNECT grant made" fi log "restoring $(basename -- "$dump") ($(stat -c %s "$dump") bytes) into $db" # One transaction: a restore that fails leaves an empty database, never a half-restored one. "${PG_AS[@]}" "$(pgbin pg_restore)" -d "$db" --exit-on-error --single-transaction < "$dump" \ || die "pg_restore failed; database $db is left empty. Drop it with --replace on the next try." log "database restored in $(( SECONDS - started )) s" fi # ------------------------------------------------------------------ assets if [[ -n "$assets" ]]; then [[ -e "$assets" ]] || die "$assets does not exist" started=$SECONDS if [[ -d "$assets" ]]; then log "copying assets from folder $assets into $assets_dir" cp -a -n -- "$assets/." "$assets_dir/" else case "$assets" in *.tar.gz|*.tgz) tar_flags="-xzf" ;; *.tar) tar_flags="-xf" ;; *) die "--assets must be a folder, .tar, .tar.gz or .tgz" ;; esac log "copying assets from $(basename -- "$assets") into $assets_dir" tar "$tar_flags" "$assets" -C "$assets_dir" --skip-old-files --no-same-owner fi if [[ "$(id -u)" == "0" ]]; then chown -R -- "$assets_owner" "$assets_dir" fi log "assets copied in $(( SECONDS - started )) s" fi # ------------------------------------------------------------------ verify assets if [[ $verify_assets -eq 1 ]]; then work="$(mktemp -d)" trap 'rm -rf -- "$work"' EXIT # sha256sum check format: " ". psql_c "$db" -At <<'SQL' > "$work/assets.sha256" SELECT encode(sha256, 'hex') || ' ' || storage_key FROM assets ORDER BY storage_key; SQL rows="$(wc -l < "$work/assets.sha256")" log "checking $rows asset rows of $db against $assets_dir" ( cd -- "$assets_dir" && sha256sum -c "$work/assets.sha256" 2>/dev/null ) > "$work/result" || true ( cd -- "$assets_dir" && find . -type f ! -path './tmp/*' | sed 's|^\./||' | LC_ALL=C sort ) > "$work/files" cut -c67- "$work/assets.sha256" | LC_ALL=C sort > "$work/rows" LC_ALL=C comm -23 "$work/files" "$work/rows" | sed 's|^|ORPHAN: |' >> "$work/result" ok="$(grep -c ': OK$' "$work/result" || true)" bad="$(grep -c ': FAILED$' "$work/result" || true)" missing="$(grep -c ': FAILED open or read$' "$work/result" || true)" orphans="$(grep -c '^ORPHAN: ' "$work/result" || true)" log "assets: $rows rows, $ok match, $bad differ, $missing missing, $orphans files with no row" grep -v ': OK$' "$work/result" >&2 || true if [[ "$ok" -ne "$rows" ]]; then die "asset check failed" fi fi # ------------------------------------------------------------------ counts if [[ -n "$db" && ( -n "$dump" || $counts_only -eq 1 ) ]]; then print_counts "$db" fi