Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/worker/jobs/backup_check.py

@@ -0,0 +1,92 @@
+"""The backup check, daily at 07:15 site time.
+
+BACKUP_DIR is this stack's backup folder on the host (/var/backups/packetstation/<stack>/), mounted read-only.
+The host's backup script writes `*.dump` files and the `last-success` and `last-failure` markers there. The
+check raises the `backup_stale` condition (subject: the stack) when the newest dump is missing, empty or older
+than 26 hours, or when `last-failure` is newer than `last-success`. Otherwise it resolves it. It only reads.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, time
+from pathlib import Path
+
+from ...feed.config import FeedSettings
+from .. import jobs
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(7, 15)
+NAME = "backup-check"
+MAX_AGE_HOURS = 26.0
+SUCCESS = "last-success"
+FAILURE = "last-failure"
+
+
+def _mtime(p: Path) -> datetime | None:
+    try:
+        return datetime.fromtimestamp(p.stat().st_mtime, UTC)
+    except OSError:
+        return None
+
+
+def inspect(folder: Path, now: datetime | None = None) -> tuple[Counts, list[str]]:
+    """(counts, problems) for a backup folder. No problems means the backups are fine."""
+    now = now or datetime.now(UTC)
+    counts: Counts = {"folder": str(folder), "dumps": 0, "newest": None, "age_hours": None, "size": None}
+    if not folder.is_dir():
+        return counts, [f"The backup folder {folder} is not there."]
+    dumps: list[tuple[datetime, Path, int]] = []
+    for p in folder.glob("*.dump"):
+        try:
+            st = p.stat()
+        except OSError:
+            continue
+        if p.is_file():
+            dumps.append((datetime.fromtimestamp(st.st_mtime, UTC), p, st.st_size))
+    counts["dumps"] = len(dumps)
+    problems: list[str] = []
+    if not dumps:
+        problems.append("No backup dump is in the folder.")
+    else:
+        at, path, size = max(dumps, key=lambda d: (d[0], d[1].name))
+        age = (now - at).total_seconds() / 3600
+        counts.update(newest=path.name, newest_at=at.isoformat(), age_hours=round(age, 1), size=size)
+        if size == 0:
+            problems.append(f"The newest dump {path.name} is empty.")
+        if age > MAX_AGE_HOURS:
+            problems.append(f"The newest dump {path.name} is {age:.0f} hours old.")
+    ok_at, fail_at = _mtime(folder / SUCCESS), _mtime(folder / FAILURE)
+    counts["last_success"] = ok_at.isoformat() if ok_at else None
+    counts["last_failure"] = fail_at.isoformat() if fail_at else None
+    if fail_at is not None and (ok_at is None or fail_at > ok_at):
+        problems.append("The last backup run failed (last-failure is newer than last-success).")
+    return counts, problems
+
+
+def run(ctx: JobContext) -> Counts:
+    stack = str(ctx.settings.stack)
+    counts, problems = inspect(Path(ctx.settings.backup_dir))
+    counts["problems"] = problems
+    title = f"Backup failed or stale ({stack})"
+    with ctx.conn.transaction():
+        if problems:
+            jobs.raise_event(
+                ctx.conn,
+                "backup_stale",
+                subject=stack,
+                title=title,
+                body="\n".join(problems),
+                detail={k: v for k, v in counts.items() if k != "problems"},
+                severity="critical",
+                condition=True,
+            )
+        else:
+            jobs.resolve_event(ctx.conn, "backup_stale", subject=stack, title=title)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)