300 lines
10 KiB
Bash
300 lines
10 KiB
Bash
 
|
#!/usr/bin/env bash
|
||
|
|
# Packet Station nightly backup. Runs on the docker host, never inside the app.
|
||
|
|
#
|
||
|
|
# For each configured stack it writes, into that stack's backup folder:
|
||
|
|
# ps-<UTC stamp>.dump pg_dump -Fc of the app database (custom format)
|
||
|
|
# ps-<UTC stamp>.globals.sql pg_dumpall --globals-only --no-role-passwords (roles, no password hashes)
|
||
|
|
# last-success / last-failure one line: "<ISO UTC time> <dump name>" (last-failure adds the reason)
|
||
|
|
#
|
||
|
|
# Each dump is written under a temporary name, checked with `pg_restore --list` run inside the
|
||
|
|
# postgres container, and only then renamed into place. The newest KEEP dumps (default 7) are kept,
|
||
|
|
# with their globals files. Any failure writes last-failure and the script exits non-zero after
|
||
|
|
# trying the remaining stacks.
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# ps-backup.sh [--config FILE] [STACK ...]
|
||
|
|
# ps-backup.sh --stack NAME CONTAINER DATABASE FOLDER [--stack ...]
|
||
|
|
#
|
||
|
|
# Config file (default /etc/packetstation/backup.conf, or PS_BACKUP_CONFIG), one stack per line:
|
||
|
|
# # name container database folder
|
||
|
|
# prod project:packetstation packetstation /var/backups/packetstation/prod
|
||
|
|
# test project:packetstation-test packetstation /home/spencerm/packetstation/test/backups
|
||
|
|
# CONTAINER is a container name, or project:<compose project> to find that project's postgres
|
||
|
|
# service by its compose labels. With STACK names given, only those stacks run.
|
||
|
|
#
|
||
|
|
# pg_dump connects inside the container as the postgres superuser over the local socket, which
|
||
|
|
# the official image trusts. No password is read, passed or printed.
|
||
|
|
#
|
||
|
|
# Globals: --no-role-passwords, so no password hash ever lands in a file (or in the PBS copy of
|
||
|
|
# it). The roles' passwords come from the stack's secrets: initdb (deploy/postgres/initdb)
|
||
|
|
# recreates them on a fresh volume. The globals file is also 0600, like the dumps.
|
||
|
|
#
|
||
|
|
# Files: dumps and globals are 0600. The markers are 0644 (a time and a file name, nothing else)
|
||
|
|
# so the worker, which mounts the folder read-only as uid 10001, can read them. When run as root
|
||
|
|
# the new files take the folder's owner and group, so a stack folder owned by spencerm stays
|
||
|
|
# readable by spencerm.
|
||
|
|
|
||
|
|
set -euo pipefail
|
||
|
|
umask 077
|
||
|
|
|
||
|
|
KEEP="${KEEP:-7}"
|
||
|
|
PG_USER="${PG_USER:-postgres}"
|
||
|
|
DOCKER="${DOCKER:-docker}"
|
||
|
|
|
||
|
|
log() { printf '%s ps-backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
|
||
|
|
|
||
|
|
usage() {
|
||
|
|
sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' >&2
|
||
|
|
exit 2
|
||
|
|
}
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------ arguments
|
||
|
|
|
||
|
|
config="${PS_BACKUP_CONFIG:-/etc/packetstation/backup.conf}"
|
||
|
|
config_given=0
|
||
|
|
declare -a only=()
|
||
|
|
declare -a stacks=() # each entry: "name|container|database|folder"
|
||
|
|
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--config)
|
||
|
|
[[ $# -ge 2 ]] || usage
|
||
|
|
config="$2"; config_given=1; shift 2 ;;
|
||
|
|
--stack)
|
||
|
|
[[ $# -ge 5 ]] || usage
|
||
|
|
stacks+=("$2|$3|$4|$5"); shift 5 ;;
|
||
|
|
--keep)
|
||
|
|
[[ $# -ge 2 ]] || usage
|
||
|
|
KEEP="$2"; shift 2 ;;
|
||
|
|
-h|--help) usage ;;
|
||
|
|
-*) log "unknown option $1"; usage ;;
|
||
|
|
*) only+=("$1"); shift ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
if ! [[ "$KEEP" =~ ^[1-9][0-9]*$ ]]; then
|
||
|
|
log "KEEP must be a positive whole number"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ ${#stacks[@]} -eq 0 ]]; then
|
||
|
|
if [[ ! -r "$config" ]]; then
|
||
|
|
log "no --stack given and the config $config is missing or unreadable"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
while read -r name container database folder extra; do
|
||
|
|
[[ -z "${name:-}" || "${name:0:1}" == "#" ]] && continue
|
||
|
|
if [[ -z "${folder:-}" || -n "${extra:-}" ]]; then
|
||
|
|
log "config line for '$name' needs exactly: name container database folder"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
stacks+=("$name|$container|$database|$folder")
|
||
|
|
done < "$config"
|
||
|
|
elif [[ $config_given -eq 1 ]]; then
|
||
|
|
log "use --config or --stack, not both"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ ${#stacks[@]} -eq 0 ]]; then
|
||
|
|
log "no stacks configured"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------ helpers
|
||
|
|
|
||
|
|
# Resolve "project:<name>" to the running postgres container of that compose project.
|
||
|
|
resolve_container() {
|
||
|
|
local spec="$1"
|
||
|
|
if [[ "$spec" == project:* ]]; then
|
||
|
|
local project="${spec#project:}" ids
|
||
|
|
ids="$("$DOCKER" ps -q \
|
||
|
|
--filter "label=com.docker.compose.project=$project" \
|
||
|
|
--filter "label=com.docker.compose.service=postgres")"
|
||
|
|
if [[ -z "$ids" ]]; then
|
||
|
|
echo "no running postgres container for compose project $project" >&2
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
if [[ "$(printf '%s\n' "$ids" | wc -l)" -ne 1 ]]; then
|
||
|
|
echo "more than one postgres container for compose project $project" >&2
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
printf '%s\n' "$ids"
|
||
|
|
else
|
||
|
|
printf '%s\n' "$spec"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# Write a marker atomically: temp file in the same folder, then rename.
|
||
|
|
write_marker() {
|
||
|
|
local folder="$1" marker="$2" text="$3" tmp
|
||
|
|
tmp="$(mktemp "$folder/.${marker}.XXXXXX")"
|
||
|
|
printf '%s\n' "$text" > "$tmp"
|
||
|
|
chmod 0644 "$tmp"
|
||
|
|
take_folder_owner "$folder" "$tmp"
|
||
|
|
mv -f "$tmp" "$folder/$marker"
|
||
|
|
}
|
||
|
|
|
||
|
|
take_folder_owner() {
|
||
|
|
local folder="$1"; shift
|
||
|
|
if [[ "$(id -u)" == "0" ]]; then
|
||
|
|
chown --reference="$folder" "$@" 2>/dev/null || true
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# Keep the newest $KEEP dumps (names sort by their UTC stamp), and drop the globals of any removed dump.
|
||
|
|
rotate() {
|
||
|
|
local folder="$1" f base
|
||
|
|
local -a dumps=()
|
||
|
|
while IFS= read -r f; do dumps+=("$f"); done < <(
|
||
|
|
find "$folder" -maxdepth 1 -type f -name 'ps-*.dump' -printf '%f\n' | LC_ALL=C sort
|
||
|
|
)
|
||
|
|
local excess=$(( ${#dumps[@]} - KEEP ))
|
||
|
|
local i
|
||
|
|
for (( i = 0; i < excess; i++ )); do
|
||
|
|
base="${dumps[$i]%.dump}"
|
||
|
|
rm -f -- "$folder/${dumps[$i]}" "$folder/$base.globals.sql"
|
||
|
|
log "removed old dump ${dumps[$i]}"
|
||
|
|
done
|
||
|
|
# Globals with no dump next to them (a run that failed after the globals step) go too.
|
||
|
|
while IFS= read -r f; do
|
||
|
|
base="${f%.globals.sql}"
|
||
|
|
[[ -e "$folder/$base.dump" ]] || rm -f -- "$folder/$f"
|
||
|
|
done < <(find "$folder" -maxdepth 1 -type f -name 'ps-*.globals.sql' -printf '%f\n')
|
||
|
|
}
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------ one stack
|
||
|
|
|
||
|
|
# Set by backup_stack for the cleanup trap. Each stack runs in its own subshell (run_stack), so
|
||
|
|
# these never leak from one stack to the next.
|
||
|
|
cur_tmp_dump=""
|
||
|
|
cur_tmp_globals=""
|
||
|
|
cur_folder=""
|
||
|
|
cur_dump_name=""
|
||
|
|
failure_written=0
|
||
|
|
|
||
|
|
cleanup_tmp() {
|
||
|
|
if [[ -n "$cur_tmp_dump" ]]; then rm -f -- "$cur_tmp_dump"; fi
|
||
|
|
if [[ -n "$cur_tmp_globals" ]]; then rm -f -- "$cur_tmp_globals"; fi
|
||
|
|
cur_tmp_dump=""
|
||
|
|
cur_tmp_globals=""
|
||
|
|
}
|
||
|
|
|
||
|
|
# Record the failure and leave the stack's subshell.
|
||
|
|
fail_stack() {
|
||
|
|
local reason="$1"
|
||
|
|
cleanup_tmp
|
||
|
|
log "FAILED: $reason"
|
||
|
|
if [[ -n "$cur_folder" && -d "$cur_folder" ]]; then
|
||
|
|
write_marker "$cur_folder" last-failure "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${cur_dump_name:--} $reason" || true
|
||
|
|
fi
|
||
|
|
failure_written=1
|
||
|
|
exit 1
|
||
|
|
}
|
||
|
|
|
||
|
|
# Any command that fails without an explicit check still ends up as last-failure.
|
||
|
|
on_stack_exit() {
|
||
|
|
local rc=$?
|
||
|
|
cleanup_tmp
|
||
|
|
if [[ $rc -ne 0 && $failure_written -eq 0 && -n "$cur_folder" && -d "$cur_folder" ]]; then
|
||
|
|
log "FAILED: unexpected error (exit $rc)"
|
||
|
|
write_marker "$cur_folder" last-failure \
|
||
|
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ) ${cur_dump_name:--} unexpected error (exit $rc)" || true
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
backup_stack() {
|
||
|
|
local name="$1" spec="$2" database="$3" folder="$4"
|
||
|
|
local stamp globals_name container started entries size
|
||
|
|
|
||
|
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||
|
|
cur_dump_name="ps-$stamp.dump"
|
||
|
|
globals_name="ps-$stamp.globals.sql"
|
||
|
|
started=$SECONDS
|
||
|
|
log "stack $name: database $database into $folder"
|
||
|
|
|
||
|
|
if [[ ! -d "$folder" ]]; then
|
||
|
|
# 0750: the worker reaches the folder through its group (see scripts/backup/README.md).
|
||
|
|
mkdir -p -- "$folder"
|
||
|
|
chmod 0750 -- "$folder" 2>/dev/null || log "stack $name: could not set mode 0750 on $folder"
|
||
|
|
fi
|
||
|
|
cur_folder="$folder"
|
||
|
|
[[ -w "$folder" ]] || fail_stack "$folder is not writable"
|
||
|
|
|
||
|
|
if command -v flock >/dev/null 2>&1; then
|
||
|
|
exec 9>"$folder/.ps-backup.lock"
|
||
|
|
flock -n 9 || fail_stack "another backup of this stack is running"
|
||
|
|
fi
|
||
|
|
|
||
|
|
container="$(resolve_container "$spec" 2>&1)" || fail_stack "$container"
|
||
|
|
|
||
|
|
cur_tmp_dump="$(mktemp "$folder/.$cur_dump_name.XXXXXX.partial")"
|
||
|
|
cur_tmp_globals="$(mktemp "$folder/.$globals_name.XXXXXX.partial")"
|
||
|
|
|
||
|
|
"$DOCKER" exec "$container" pg_dump -U "$PG_USER" -d "$database" -Fc > "$cur_tmp_dump" \
|
||
|
|
|| fail_stack "pg_dump failed"
|
||
|
|
[[ -s "$cur_tmp_dump" ]] || fail_stack "pg_dump wrote nothing"
|
||
|
|
|
||
|
|
"$DOCKER" exec "$container" pg_dumpall -U "$PG_USER" --globals-only --no-role-passwords \
|
||
|
|
> "$cur_tmp_globals" || fail_stack "pg_dumpall --globals-only failed"
|
||
|
|
[[ -s "$cur_tmp_globals" ]] || fail_stack "pg_dumpall wrote nothing"
|
||
|
|
|
||
|
|
# The check reads the dump from stdin with the container's own pg_restore. Entry lines start
|
||
|
|
# with the entry's id; comment lines start with a semicolon.
|
||
|
|
entries="$("$DOCKER" exec -i "$container" pg_restore --list < "$cur_tmp_dump" | grep -c '^[0-9]')" \
|
||
|
|
|| fail_stack "pg_restore --list could not read the dump"
|
||
|
|
[[ "$entries" -ge 1 ]] || fail_stack "pg_restore --list found no entries"
|
||
|
|
|
||
|
|
chmod 0600 "$cur_tmp_dump" "$cur_tmp_globals"
|
||
|
|
take_folder_owner "$folder" "$cur_tmp_dump" "$cur_tmp_globals"
|
||
|
|
mv -f -- "$cur_tmp_globals" "$folder/$globals_name"
|
||
|
|
cur_tmp_globals=""
|
||
|
|
mv -f -- "$cur_tmp_dump" "$folder/$cur_dump_name"
|
||
|
|
cur_tmp_dump=""
|
||
|
|
|
||
|
|
size="$(stat -c %s "$folder/$cur_dump_name")"
|
||
|
|
write_marker "$folder" last-success "$(date -u +%Y-%m-%dT%H:%M:%SZ) $cur_dump_name"
|
||
|
|
rotate "$folder"
|
||
|
|
log "stack $name: ok, $cur_dump_name, $size bytes, $entries entries, $(( SECONDS - started )) s"
|
||
|
|
}
|
||
|
|
|
||
|
|
# One stack in its own subshell with errexit on, so a failure ends that stack only.
|
||
|
|
run_stack() (
|
||
|
|
set -euo pipefail
|
||
|
|
trap on_stack_exit EXIT
|
||
|
|
backup_stack "$@"
|
||
|
|
)
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------ main
|
||
|
|
|
||
|
|
failures=0
|
||
|
|
ran=0
|
||
|
|
for entry in "${stacks[@]}"; do
|
||
|
|
IFS='|' read -r s_name s_container s_database s_folder <<< "$entry"
|
||
|
|
if [[ ${#only[@]} -gt 0 ]]; then
|
||
|
|
wanted=0
|
||
|
|
for o in "${only[@]}"; do
|
||
|
|
if [[ "$o" == "$s_name" ]]; then wanted=1; fi
|
||
|
|
done
|
||
|
|
[[ $wanted -eq 1 ]] || continue
|
||
|
|
fi
|
||
|
|
ran=$(( ran + 1 ))
|
||
|
|
# Not in an if or ||: that would switch errexit off inside the subshell.
|
||
|
|
set +e
|
||
|
|
run_stack "$s_name" "$s_container" "$s_database" "$s_folder"
|
||
|
|
rc=$?
|
||
|
|
set -e
|
||
|
|
if [[ $rc -ne 0 ]]; then
|
||
|
|
failures=$(( failures + 1 ))
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
if [[ $ran -eq 0 ]]; then
|
||
|
|
log "no configured stack matched: ${only[*]}"
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
if [[ $failures -gt 0 ]]; then
|
||
|
|
log "$failures of $ran stacks failed"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
log "all $ran stacks backed up"
|