spencerm/PacketStation
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dad21db65f26428f03c3437fc71ddb24d5057c26
PacketStation/deploy/vm/install.sh
T

511 lines
20 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Packet Station native VM install. Run by Spencer as root on a new test or production VM, from an
# unpacked release (make-release.sh). Idempotent: run it again after a change and it only fixes
# what differs. It never overwrites the env file or a secret, never starts the app or the worker,
# and never touches VISUAL, M-Files or AD.
#
# Usage:
# install.sh --stack test|prod [--ca-bundle FILE] [--generate-secrets] [--skip-packages]
# install.sh --files-only
#
# --stack S test or prod. Picks the env file example. Read from the env file when it exists.
# --ca-bundle FILE the PESCO CA chain (PEM, scripts/Export-PescoCaBundle.ps1). Installed as
# /etc/packetstation/ca/pesco-ca.pem and into the system trust store.
# --generate-secrets create the four internal secrets that are missing (both PostgreSQL role
# passwords, session_key, seal_key) from /dev/urandom. Never replaces a file.
# --skip-packages leave apt alone (repositories and packages already in place).
# --files-only only the systemd units, the ps-* commands, backup.conf and the firewall
# table. ps-upgrade runs this from the new release.
#
# What it does, in order:
# 1. apt: PostgreSQL 16 (PGDG), ODBC Driver 18 (Microsoft), Python 3.12, nftables, nfs-common.
# 2. User and group packetstation, uid and gid 10001 (the image's uid, and the NFS share's).
# 3. Folders: /etc/packetstation, /var/lib/packetstation/assets, /var/backups/packetstation,
# /opt/packetstation, the archive mount point /mnt/packetstation-archive.
# 4. /etc/packetstation/packetstation.env from the stack's example, if missing.
# 5. The PESCO CA, when --ca-bundle is given.
# 6. Secret files: owner root:packetstation, mode 0440, one line, no carriage return.
# 7. PostgreSQL: socket only (conf.d/packetstation.conf), scram for ps_app and ps_migrate on the
# socket (pg_hba.conf), the roles and the database as deploy/postgres/initdb makes them.
# 8. Units, commands, backup timer, firewall (the files-only part).
set -euo pipefail
umask 022
PS_USER=packetstation
PS_ID=10001
PG_VERSION=16
PG_CLUSTER=main
ETC=/etc/packetstation
SECRETS="$ETC/secrets"
ENV_FILE="$ETC/packetstation.env"
CA_FILE="$ETC/ca/pesco-ca.pem"
NFT_FILE="$ETC/packetstation.nft"
BACKUP_CONF="$ETC/backup.conf"
ASSETS=/var/lib/packetstation/assets
BACKUPS=/var/backups/packetstation
ARCHIVE_MOUNT=/mnt/packetstation-archive
OPT=/opt/packetstation
SBIN=/usr/local/sbin
UNIT_DIR=/etc/systemd/system
PG_ETC="/etc/postgresql/$PG_VERSION/$PG_CLUSTER"
PG_UNIT="postgresql@$PG_VERSION-$PG_CLUSTER.service"
PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin"
DB_LOCALE=en_US.UTF-8
# Signing keys, checked by fingerprint before apt trusts them.
PGDG_KEY_URL=https://www.postgresql.org/media/keys/ACCC4CF8.asc
PGDG_KEY_FPR=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
MS_KEY_URL=https://packages.microsoft.com/keys/microsoft.asc
MS_KEY_FPR=BC528686B50D79E339D3721CEB3E94ADBE1229CF
ALL_SECRETS=(postgres_migrate_password postgres_app_password visualkit_password mfileskit_password
ldap_bind_password session_key seal_key)
GENERATED_SECRETS=(postgres_migrate_password postgres_app_password session_key seal_key)
here="$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" && pwd -P)"
release="$(cd -- "$here/../.." && pwd -P)"
log() { printf 'install: %s\n' "$*" >&2; }
die() { printf 'install: ERROR: %s\n' "$*" >&2; exit 1; }
usage() { sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
stack=""
ca_bundle=""
generate=0
skip_packages=0
files_only=0
while [[ $# -gt 0 ]]; do
case "$1" in
--stack) [[ $# -ge 2 ]] || usage; stack="$2"; shift 2 ;;
--ca-bundle) [[ $# -ge 2 ]] || usage; ca_bundle="$2"; shift 2 ;;
--generate-secrets) generate=1; shift ;;
--skip-packages) skip_packages=1; shift ;;
--files-only) files_only=1; shift ;;
-h|--help) usage ;;
*) log "unknown argument $1"; usage ;;
esac
done
[[ "$(id -u)" == "0" ]] || die "run as root"
[[ -z "$stack" || "$stack" == "test" || "$stack" == "prod" ]] || die "--stack must be test or prod"
# ------------------------------------------------------------------ helpers
# A KEY=value from the env file, with surrounding quotes removed. Empty when unset.
env_value() {
local key="$1" line value
[[ -r "$ENV_FILE" ]] || return 0
line="$(grep -E "^[[:space:]]*$key=" "$ENV_FILE" | tail -n 1 || true)"
value="${line#*=}"
value="${value%$'\r'}"
value="${value#\"}"; value="${value%\"}"
value="${value#\'}"; value="${value%\'}"
printf '%s' "$value"
}
# Install a text file only when it differs. Strips carriage returns (a Windows checkout).
# Prints "changed" when it wrote the file.
put_file() {
local src="$1" dst="$2" owner="$3" mode="$4" tmp
tmp="$(mktemp)"
sed 's/\r$//' -- "$src" > "$tmp"
if [[ -f "$dst" ]] && cmp -s -- "$tmp" "$dst"; then
chown "$owner" -- "$dst"; chmod "$mode" -- "$dst"
rm -f -- "$tmp"
return 0
fi
install -o "${owner%%:*}" -g "${owner##*:}" -m "$mode" -- "$tmp" "$dst"
rm -f -- "$tmp"
echo changed
}
as_postgres() {
setpriv --reuid=postgres --regid=postgres --init-groups -- "$@"
}
is_ipv4() { [[ "$1" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; }
is_ipv6() { [[ "$1" =~ ^[0-9A-Fa-f:]+$ && "$1" == *:* ]]; }
# ------------------------------------------------------------------ 1. packages
os_id=""
os_version=""
os_codename=""
read_os() {
# shellcheck disable=SC1091
os_id="$(. /etc/os-release && printf '%s' "${ID:-}")"
# shellcheck disable=SC1091
os_version="$(. /etc/os-release && printf '%s' "${VERSION_ID:-}")"
# shellcheck disable=SC1091
os_codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")"
}
fetch_key() {
local url="$1" fpr="$2" dst="$3" tmp found
tmp="$(mktemp -d)"
curl -fsSL --proto '=https' "$url" -o "$tmp/key.asc"
found="$(gpg --batch --show-keys --with-colons "$tmp/key.asc" 2>/dev/null | awk -F: '$1 == "fpr" && !seen { print $10; seen = 1 }')"
if [[ "$found" != "$fpr" ]]; then
rm -rf -- "$tmp"
die "the key from $url has fingerprint '${found:-none}', expected $fpr. Check it before going on."
fi
gpg --batch --yes --dearmor -o "$tmp/key.gpg" "$tmp/key.asc"
install -o root -g root -m 0644 "$tmp/key.gpg" "$dst"
rm -rf -- "$tmp"
}
step_packages() {
local ms_repo python_pkgs=()
case "$os_id $os_version" in
"ubuntu 24.04")
ms_repo="https://packages.microsoft.com/ubuntu/24.04/prod $os_codename main"
python_pkgs=(python3.12 python3.12-venv) ;;
"debian 12")
ms_repo="https://packages.microsoft.com/debian/12/prod $os_codename main"
log "Debian 12 ships Python 3.11. Packet Station needs 3.12: give ps-upgrade PYTHON=/path/to/python3.12." ;;
*) die "unsupported OS '$os_id $os_version'. Supported: Ubuntu 24.04, Debian 12." ;;
esac
export DEBIAN_FRONTEND=noninteractive
log "apt: base packages"
apt-get update -q
apt-get install -y -q --no-install-recommends \
ca-certificates curl gnupg locales tzdata nftables nfs-common util-linux coreutils findutils
# The database locale. Generated before PostgreSQL so its first cluster knows it too.
if ! locale -a 2>/dev/null | grep -qiE '^en_US\.utf-?8$'; then
log "generating locale $DB_LOCALE"
if [[ -f /etc/locale.gen ]]; then
sed -i 's/^# *\(en_US.UTF-8 UTF-8\)/\1/' /etc/locale.gen
fi
locale-gen en_US.UTF-8 >/dev/null
fi
log "apt: PGDG and Microsoft repositories"
fetch_key "$PGDG_KEY_URL" "$PGDG_KEY_FPR" /usr/share/keyrings/pgdg.gpg
fetch_key "$MS_KEY_URL" "$MS_KEY_FPR" /usr/share/keyrings/microsoft-prod.gpg
printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/pgdg.gpg] https://apt.postgresql.org/pub/repos/apt %s-pgdg main\n' \
"$os_codename" > /etc/apt/sources.list.d/pgdg.list
printf 'deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] %s\n' "$ms_repo" \
> /etc/apt/sources.list.d/microsoft-prod.list
apt-get update -q
log "apt: PostgreSQL $PG_VERSION, ODBC Driver 18, Python"
ACCEPT_EULA=Y apt-get install -y -q --no-install-recommends \
"postgresql-$PG_VERSION" "postgresql-client-$PG_VERSION" msodbcsql18 unixodbc "${python_pkgs[@]}"
odbcinst -q -d -n "ODBC Driver 18 for SQL Server" >/dev/null \
|| die "ODBC Driver 18 for SQL Server is not registered with unixODBC"
}
# ------------------------------------------------------------------ 2. user
step_user() {
local g u
g="$(getent group "$PS_USER" | cut -d: -f3 || true)"
if [[ -z "$g" ]]; then
if getent group "$PS_ID" >/dev/null; then
die "gid $PS_ID is taken by group $(getent group "$PS_ID" | cut -d: -f1)"
fi
groupadd --system --gid "$PS_ID" "$PS_USER"
log "created group $PS_USER ($PS_ID)"
elif [[ "$g" != "$PS_ID" ]]; then
die "group $PS_USER has gid $g, expected $PS_ID (the NFS share and the backups expect $PS_ID)"
fi
u="$(id -u "$PS_USER" 2>/dev/null || true)"
if [[ -z "$u" ]]; then
if getent passwd "$PS_ID" >/dev/null; then
die "uid $PS_ID is taken by user $(getent passwd "$PS_ID" | cut -d: -f1)"
fi
useradd --system --uid "$PS_ID" --gid "$PS_ID" --home-dir /var/lib/packetstation --no-create-home \
--shell /usr/sbin/nologin --comment "Packet Station" "$PS_USER"
log "created user $PS_USER ($PS_ID)"
elif [[ "$u" != "$PS_ID" ]]; then
die "user $PS_USER has uid $u, expected $PS_ID"
fi
}
# ------------------------------------------------------------------ 3. folders
step_dirs() {
install -d -o root -g "$PS_USER" -m 0750 "$ETC" "$SECRETS"
install -d -o root -g root -m 0755 "$ETC/ca" "$OPT"
install -d -o root -g "$PS_USER" -m 0750 /var/lib/packetstation "$BACKUPS"
install -d -o "$PS_USER" -g "$PS_USER" -m 0750 "$ASSETS"
# The mount point stays root's and read-only to packetstation, so while the share is not mounted
# nothing can be written into the VM's own disk by mistake.
if mountpoint -q "$ARCHIVE_MOUNT" 2>/dev/null; then
log "$ARCHIVE_MOUNT is mounted; left as it is"
else
install -d -o root -g root -m 0755 "$ARCHIVE_MOUNT"
fi
}
# ------------------------------------------------------------------ 4. env file
step_env() {
local example have
if [[ -e "$ENV_FILE" ]]; then
have="$(env_value STACK)"
if [[ -n "$stack" && "$have" != "$stack" ]]; then
die "$ENV_FILE says STACK=$have, but --stack $stack was given"
fi
chown root:"$PS_USER" "$ENV_FILE"; chmod 0640 "$ENV_FILE"
log "kept $ENV_FILE (STACK=$have)"
return 0
fi
[[ -n "$stack" ]] || die "no $ENV_FILE yet: give --stack test or --stack prod"
example="$here/$stack.env.example"
[[ -r "$example" ]] || die "missing $example"
put_file "$example" "$ENV_FILE" "root:$PS_USER" 0640 >/dev/null
log "wrote $ENV_FILE from $(basename -- "$example"). Read it through before the first start."
}
# ------------------------------------------------------------------ 5. CA
step_ca() {
if [[ -z "$ca_bundle" ]]; then
[[ -f "$CA_FILE" ]] || log "no PESCO CA yet: run again with --ca-bundle FILE (LDAP and SMTP need it)"
return 0
fi
[[ -r "$ca_bundle" ]] || die "cannot read $ca_bundle"
grep -q 'BEGIN CERTIFICATE' "$ca_bundle" || die "$ca_bundle holds no PEM certificate"
if grep -q 'PRIVATE KEY' "$ca_bundle"; then
die "$ca_bundle holds a private key. Export only the certificates."
fi
put_file "$ca_bundle" "$CA_FILE" root:root 0644 >/dev/null
# update-ca-certificates wants one certificate per .crt file.
local dir=/usr/local/share/ca-certificates/pesco tmp
tmp="$(mktemp -d)"
sed 's/\r$//' -- "$CA_FILE" | awk -v d="$tmp" '
/-----BEGIN CERTIFICATE-----/ { n++; f = sprintf("%s/pesco-ca-%02d.crt", d, n) }
f { print > f }
/-----END CERTIFICATE-----/ { close(f); f = "" }'
rm -rf -- "$dir"
install -d -o root -g root -m 0755 "$dir"
install -o root -g root -m 0644 "$tmp"/*.crt "$dir"/
rm -rf -- "$tmp"
update-ca-certificates >/dev/null
log "PESCO CA installed: $CA_FILE and the system trust store ($(find "$dir" -name '*.crt' | wc -l) certificates)"
}
# ------------------------------------------------------------------ 6. secrets
new_secret() {
# 32 random bytes as 64 hex characters. Held in a variable and written by a builtin, never on argv.
local name="$1" value tmp
value="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')"
[[ ${#value} -eq 64 ]] || die "could not read /dev/urandom"
tmp="$(mktemp "$SECRETS/.$name.XXXXXX")"
printf '%s' "$value" > "$tmp"
unset value
chown root:"$PS_USER" "$tmp"; chmod 0440 "$tmp"
mv -- "$tmp" "$SECRETS/$name"
log "generated secret $name"
}
missing_secrets=()
step_secrets() {
local name f g
missing_secrets=()
for name in "${ALL_SECRETS[@]}"; do
f="$SECRETS/$name"
if [[ ! -e "$f" && $generate -eq 1 ]]; then
for g in "${GENERATED_SECRETS[@]}"; do
if [[ "$g" == "$name" ]]; then new_secret "$name"; fi
done
fi
if [[ ! -e "$f" ]]; then
missing_secrets+=("$name")
continue
fi
chown root:"$PS_USER" -- "$f"; chmod 0440 -- "$f"
[[ -s "$f" ]] || die "secret $name is empty"
if [[ "$(wc -l < "$f")" -gt 1 ]] || grep -q $'\r' -- "$f"; then
die "secret $name must be one line with no carriage return"
fi
done
if [[ ${#missing_secrets[@]} -gt 0 ]]; then
log "secrets still missing in $SECRETS: ${missing_secrets[*]}"
fi
}
have_secret() { [[ -s "$SECRETS/$1" ]]; }
# ------------------------------------------------------------------ 7. PostgreSQL
step_postgres() {
local changed="" hba="$PG_ETC/pg_hba.conf" tmp
[[ -d "$PG_ETC" ]] || die "no PostgreSQL cluster $PG_VERSION/$PG_CLUSTER in $PG_ETC"
grep -Eq "^[[:space:]]*include_dir[[:space:]]*=[[:space:]]*'conf\.d'" "$PG_ETC/postgresql.conf" \
|| die "$PG_ETC/postgresql.conf has no include_dir = 'conf.d'"
install -d -o postgres -g postgres -m 0755 "$PG_ETC/conf.d"
changed="$(put_file "$here/postgres/packetstation.conf" "$PG_ETC/conf.d/packetstation.conf" postgres:postgres 0644)"
# ps_app and ps_migrate log in on the socket with their passwords (scram). The block goes first,
# before Debian's "local all all peer", which would refuse them (the OS user is packetstation).
if ! grep -q '^# packetstation begin' "$hba"; then
tmp="$(mktemp)"
awk '
!done && $0 !~ /^[[:space:]]*(#|$)/ {
print "# packetstation begin (deploy/vm/install.sh)"
print "local packetstation ps_app,ps_migrate scram-sha-256"
print "# packetstation end"
print ""
done = 1
}
{ print }' "$hba" > "$tmp"
install -o postgres -g postgres -m 0640 "$tmp" "$hba"
rm -f -- "$tmp"
log "pg_hba.conf: ps_app and ps_migrate use scram-sha-256 on the socket"
[[ -n "$changed" ]] || changed=reload
fi
systemctl enable --quiet "$PG_UNIT" 2>/dev/null || true
if [[ "$changed" == "changed" ]]; then
log "restarting PostgreSQL (conf.d/packetstation.conf changed)"
systemctl restart "$PG_UNIT"
elif [[ "$changed" == "reload" ]]; then
systemctl start "$PG_UNIT"
systemctl reload "$PG_UNIT"
else
systemctl start "$PG_UNIT"
fi
if ! have_secret postgres_migrate_password || ! have_secret postgres_app_password; then
log "skipping roles and database: the PostgreSQL secrets are missing (or use --generate-secrets)"
return 0
fi
log "PostgreSQL roles ps_migrate and ps_app, database packetstation"
# The passwords reach psql through its environment (\getenv), never through argv or a file.
# A failed statement is not logged with its text (log_min_error_statement), so a password
# cannot land in the server log.
PS_MIGRATE_PW="$(<"$SECRETS/postgres_migrate_password")"
PS_APP_PW="$(<"$SECRETS/postgres_app_password")"
export PS_MIGRATE_PW PS_APP_PW
as_postgres "$PG_BIN/psql" -X -q -v ON_ERROR_STOP=1 -d postgres <<SQL
SET log_min_error_statement = panic;
SET log_statement = 'none';
\\getenv migrate_pw PS_MIGRATE_PW
\\getenv app_pw PS_APP_PW
SELECT 'CREATE ROLE ps_migrate' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_migrate') \\gexec
SELECT 'CREATE ROLE ps_app' WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ps_app') \\gexec
ALTER ROLE ps_migrate WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'migrate_pw';
ALTER ROLE ps_app WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD :'app_pw';
\\unset migrate_pw
\\unset app_pw
SELECT format('CREATE DATABASE packetstation OWNER ps_migrate ENCODING %L LOCALE %L TEMPLATE template0',
'UTF8', '$DB_LOCALE')
WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'packetstation') \\gexec
REVOKE ALL ON DATABASE packetstation FROM PUBLIC;
GRANT CONNECT, TEMPORARY ON DATABASE packetstation TO ps_app;
\\connect packetstation
REVOKE ALL ON SCHEMA public FROM PUBLIC;
ALTER SCHEMA public OWNER TO ps_migrate;
SQL
unset PS_MIGRATE_PW PS_APP_PW
}
# ------------------------------------------------------------------ 8. files
write_firewall() {
local proxy rule tmp
proxy="$(env_value TRUSTED_PROXY_HOST)"
if is_ipv4 "$proxy"; then
rule="ip saddr $proxy"
elif is_ipv6 "$proxy"; then
rule="ip6 saddr $proxy"
else
die "TRUSTED_PROXY_HOST in $ENV_FILE must be NPM's IP address (it is '${proxy}')"
fi
tmp="$(mktemp)"
cat > "$tmp" <<NFT
#!/usr/sbin/nft -f
# Written by deploy/vm/install.sh from TRUSTED_PROXY_HOST in $ENV_FILE. Do not edit:
# change the env file and run install.sh --files-only. Loaded by packetstation-firewall.service.
# Port 8000 (the app) answers only NPM on docker-1 and the VM itself. Nothing else is filtered here.
table inet packetstation
delete table inet packetstation
table inet packetstation {
chain input {
type filter hook input priority filter; policy accept;
tcp dport 8000 iifname "lo" accept
tcp dport 8000 $rule accept
tcp dport 8000 counter drop
}
}
NFT
if command -v nft >/dev/null 2>&1; then
nft -c -f "$tmp" || { rm -f -- "$tmp"; die "nft refused the generated table"; }
fi
put_file "$tmp" "$NFT_FILE" root:root 0644 >/dev/null
rm -f -- "$tmp"
}
step_files() {
local f st
[[ -r "$ENV_FILE" ]] || die "no $ENV_FILE: run install.sh --stack test|prod first"
st="$(env_value STACK)"
[[ "$st" == "test" || "$st" == "prod" ]] || die "STACK in $ENV_FILE must be test or prod"
for f in "$here"/systemd/*.service; do
put_file "$f" "$UNIT_DIR/$(basename -- "$f")" root:root 0644 >/dev/null
done
put_file "$release/scripts/backup/packetstation-backup.service" "$UNIT_DIR/packetstation-backup.service" root:root 0644 >/dev/null
put_file "$release/scripts/backup/packetstation-backup.timer" "$UNIT_DIR/packetstation-backup.timer" root:root 0644 >/dev/null
put_file "$here/ps-run.sh" "$SBIN/ps-run" root:root 0755 >/dev/null
put_file "$here/ps-doctor.sh" "$SBIN/ps-doctor" root:root 0755 >/dev/null
put_file "$here/upgrade.sh" "$SBIN/ps-upgrade" root:root 0755 >/dev/null
put_file "$release/scripts/backup/ps-backup.sh" "$SBIN/ps-backup" root:root 0755 >/dev/null
put_file "$release/scripts/backup/ps-restore.sh" "$SBIN/ps-restore" root:root 0755 >/dev/null
if [[ ! -e "$BACKUP_CONF" ]]; then
printf '# name database folder\n%s packetstation %s\n' "$st" "$BACKUPS" > "$BACKUP_CONF"
chmod 0644 "$BACKUP_CONF"
log "wrote $BACKUP_CONF"
fi
write_firewall
systemctl daemon-reload
systemctl enable --quiet packetstation-firewall.service packetstation-backup.timer
if systemctl is-active --quiet packetstation-firewall.service; then
systemctl reload packetstation-firewall.service
else
systemctl start packetstation-firewall.service
fi
systemctl start packetstation-backup.timer
log "units, ps-run, ps-doctor, ps-upgrade, ps-backup, ps-restore and the firewall table are in place"
}
# ------------------------------------------------------------------ main
if [[ $files_only -eq 1 ]]; then
step_files
exit 0
fi
read_os
if [[ $skip_packages -eq 0 ]]; then
step_packages
fi
for tool in setpriv systemctl install mountpoint; do
command -v "$tool" >/dev/null 2>&1 || die "$tool is not installed"
done
step_user
step_dirs
step_env
step_ca
step_secrets
step_postgres
step_files
cat >&2 <<EOF
install: done for STACK=$(env_value STACK).
install: next:
- read $ENV_FILE through
- secrets still missing: ${missing_secrets[*]:-none}
- install a release: ps-upgrade /path/to/packetstation-<build>.tar.gz
- the app and the worker are not enabled or started. Starting them is a separate step (docs/DEPLOY.md).
EOF
Reference in New Issue View Git Blame Copy Permalink