Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/server/test_admin_security.py

@@ -0,0 +1,236 @@
+"""Admin Security: alerts, the integrity result, the access log and its CSV, who is in, the stamp check."""
+
+from __future__ import annotations
+
+import csv
+import io
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from fakedir import FakeDirectory
+from fastapi.testclient import TestClient
+from pshelpers import PIN, app_conn, needs_pg, pin_signin, register_device, set_pin, sign_in
+from psycopg.types.json import Jsonb
+
+from packetstation.auth import ratelimit
+from packetstation.pdf import stamps
+from packetstation.settings import Settings
+
+pytestmark = needs_pg
+
+FORMULA = "=cmd|' /C calc'!A0"
+
+
+def test_access_log_filters_pages_and_csv(
+    pg_client: TestClient, pg_settings: Settings, fake_dir: FakeDirectory
+) -> None:
+    assert sign_in(pg_client, "clerk.clara").status_code == 200
+    assert sign_in(pg_client, "clerk.clara", "wrong").status_code == 401
+    assert sign_in(pg_client, FORMULA, "wrong").status_code == 401
+    clara = str(fake_dir.user("clerk.clara").guid)
+    with app_conn(pg_settings) as conn:
+        ratelimit.record(conn, account="wh.wade", ip="10.1.2.3", kind="password", ok=False, reason="bad_password")
+    sign_in(pg_client, "admin.ann")
+
+    rows = pg_client.get("/api/admin/access-log").json()["rows"]
+    sources = {r["source"] for r in rows}
+    assert sources == {"attempt", "audit"}
+    assert all(
+        set(r) == {"id", "at", "source", "kind", "ok", "account", "user_id", "display_name", "ip", "detail"}
+        for r in rows
+    )
+    failed = pg_client.get("/api/admin/access-log", params={"ok": "false", "source": "x"}).json()["rows"]
+    assert failed and all(r["ok"] is False for r in failed)
+    assert {r["account"] for r in failed if r["source"] == "attempt"} == {"clerk.clara", FORMULA.lower(), "wh.wade"}
+    mine = pg_client.get("/api/admin/access-log", params={"user_id": clara}).json()["rows"]
+    assert mine and all(r["user_id"] == clara for r in mine)
+    assert {r["kind"] for r in mine} >= {"password", "auth.signin"}
+    q = pg_client.get("/api/admin/access-log", params={"q": "Clara"}).json()["rows"]
+    assert q and all(r["display_name"] == "Clara Clerk" or r["account"] == "clerk.clara" for r in q)
+    kind = pg_client.get("/api/admin/access-log", params={"kind": "auth.signin"}).json()["rows"]
+    assert kind and all(r["kind"] == "auth.signin" and r["source"] == "audit" for r in kind)
+    ip = pg_client.get("/api/admin/access-log", params={"ip": "10.1.2.0/24"}).json()["rows"]
+    assert [r["account"] for r in ip] == ["wh.wade"] and ip[0]["ip"] == "10.1.2.3"
+    assert pg_client.get("/api/admin/access-log", params={"ip": "nope"}).status_code == 422
+    future = (datetime.now(UTC) + timedelta(hours=1)).isoformat()
+    assert pg_client.get("/api/admin/access-log", params={"since": future}).json()["rows"] == []
+    assert pg_client.get("/api/admin/access-log", params={"until": future}).json()["rows"]
+
+    # Pages: no row twice, none missed (attempt and audit rows share a timestamp inside one transaction)
+    every = pg_client.get("/api/admin/access-log", params={"limit": 1000}).json()
+    assert every["next_before"] is None
+    seen: list[str] = []
+    before = ""
+    while True:
+        page = pg_client.get("/api/admin/access-log", params={"limit": 3, "before": before}).json()
+        seen += [r["id"] for r in page["rows"]]
+        if not page["next_before"]:
+            break
+        before = page["next_before"]
+    assert seen == [r["id"] for r in every["rows"]]
+    assert pg_client.get("/api/admin/access-log", params={"before": "garbage"}).status_code == 422
+
+    # The CSV: the same rows, the formula guard, logged and audited
+    r = pg_client.get("/api/admin/access-log.csv", params={"ok": "false"})
+    assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv")
+    assert "attachment" in r.headers["content-disposition"]
+    table = list(csv.reader(io.StringIO(r.text)))
+    assert table[0] == ["at", "source", "kind", "ok", "account", "display_name", "user_id", "ip", "detail"]
+    accounts = [row[4] for row in table[1:]]
+    assert "'" + FORMULA.lower() in accounts and FORMULA.lower() not in accounts
+    assert all(row[3] == "no" for row in table[1:])
+    with app_conn(pg_settings) as conn:
+        log = conn.execute("SELECT kind, what FROM print_log").fetchall()
+        assert log == [("access-log-csv", f"Access log CSV ({len(table) - 1} rows)")]
+        detail = conn.execute("SELECT detail FROM audit_events WHERE action = 'admin.access_log_exported'").fetchone()[
+            0
+        ]
+        assert detail["rows"] == len(table) - 1 and detail["filters"] == {"ok": "False"}
+
+
+def _print_row(settings: Settings, user_id: Any, *, sig_override: str | None = None) -> tuple[int, str, str]:
+    who = {"id": str(user_id), "display_name": "Ann Admin", "emp_no": "1001"}
+    now = datetime.now(UTC)
+    text = stamps.controlled_stamp(who, now, mode="report")
+    line, sig = stamps.signed_stamp(settings, text, packet_id=None, kind="report", seal_hash_hex=None, who=who, now=now)
+    if sig_override is not None:
+        line, sig = stamps.with_sig(text, sig_override), sig_override
+    with app_conn(settings) as conn:
+        pid = conn.execute(
+            "INSERT INTO print_log (by, by_name, kind, what, pages, stamp, stamp_sig)"
+            " VALUES (%s, 'Ann Admin', 'report-pdf', 'Production status report', 2, %s, %s) RETURNING id",
+            (user_id, line, sig),
+        ).fetchone()[0]
+    return int(pid), line, sig
+
+
+def _pdf(text: str) -> bytes:
+    from reportlab.pdfgen.canvas import Canvas
+
+    buf = io.BytesIO()
+    c = Canvas(buf)
+    c.setFont("Helvetica", 6)
+    c.drawString(20, 20, text)
+    c.drawString(20, 60, "Some other text on the page")
+    c.save()
+    return buf.getvalue()
+
+
+def test_stamp_verify(pg_client: TestClient, pg_settings: Settings, fake_dir: FakeDirectory) -> None:
+    sign_in(pg_client, "admin.ann")
+    ann = fake_dir.user("admin.ann").guid
+    pid, line, sig = _print_row(pg_settings, ann)
+
+    def check(text: str) -> dict[str, Any]:
+        r = pg_client.post("/api/admin/stamp-verify", json={"text": text})
+        assert r.status_code == 200, r.text
+        out: dict[str, Any] = r.json()
+        return out
+
+    ok = check(line)
+    assert ok["found"] is True and ok["valid"] is True, ok
+    assert ok["print"] == {
+        "id": pid,
+        "at": ok["print"]["at"],
+        "by_name": "Ann Admin",
+        "kind": "report-pdf",
+        "what": "Production status report",
+        "pages": 2,
+        "packet_id": None,
+        "packet_label": None,
+    }
+    assert check(sig)["valid"] is True  # the 16 hex alone
+    assert check(sig.upper())["valid"] is True
+    assert check("  " + line.replace(" - ", "  -  ") + "\n")["valid"] is True  # spacing does not matter
+    tail = line[line.index("printed by") :]
+    assert check(tail)["valid"] is True  # a partial copy from the end
+
+    tampered = check(line.replace("Ann Admin", "Bob Admin"))
+    assert tampered["found"] is True and tampered["valid"] is False and "differs" in tampered["reason"]
+    other_sig = ("0" if sig[0] != "0" else "1") + sig[1:]
+    unknown = check(line.replace(sig, other_sig))
+    assert unknown["found"] is False and unknown["valid"] is False and unknown["print"] is None
+    nothing = check("no stamp in here")
+    assert nothing["found"] is False and nothing["reason"] == "No stamp signature in the text."
+
+    # A print_log row whose signature was never made with the seal key
+    _, forged_line, _ = _print_row(pg_settings, ann, sig_override="deadbeefdeadbeef")
+    forged = check(forged_line)
+    assert forged["found"] is True and forged["valid"] is False and "does not match" in forged["reason"]
+
+    # From a PDF
+    r = pg_client.post("/api/admin/stamp-verify/pdf", files={"file": ("copy.pdf", _pdf(line), "application/pdf")})
+    assert r.status_code == 200, r.text
+    found = r.json()["stamps"]
+    assert len(found) == 1 and found[0]["valid"] is True and found[0]["print"]["id"] == pid
+    r = pg_client.post(
+        "/api/admin/stamp-verify/pdf",
+        files={"file": ("copy.pdf", _pdf(line.replace("Ann Admin", "Bob Admin")), "application/pdf")},
+    )
+    assert r.json()["stamps"][0]["valid"] is False
+    r = pg_client.post("/api/admin/stamp-verify/pdf", files={"file": ("x.pdf", b"not a pdf", "application/pdf")})
+    assert r.status_code == 422 and r.json()["code"] == "not_pdf"
+
+    with app_conn(pg_settings) as conn:
+        n = conn.execute("SELECT count(*) FROM audit_events WHERE action = 'admin.stamp_verified'").fetchone()[0]
+        assert n == 11
+
+
+def test_alerts_integrity_and_presence(pg_client: TestClient, pg_settings: Settings, fake_dir: FakeDirectory) -> None:
+    # Three wrong PINs at sign-in, then a session ended after three wrong unlock PINs
+    register_device(pg_client)
+    set_pin(pg_client, "weld.andy")
+    for _ in range(3):
+        assert pin_signin(pg_client, "3003", "000000").status_code == 401
+    with app_conn(pg_settings) as conn:
+        conn.execute("UPDATE user_pins SET failed_count = 0")
+    assert pin_signin(pg_client, "3003", PIN).status_code == 200
+    pg_client.post("/api/auth/lock")
+    for _ in range(3):
+        pg_client.post("/api/auth/unlock", json={"factor": "pin", "secret": "111111"})
+    # A denial
+    sign_in(pg_client, "clerk.clara")
+    assert pg_client.get("/api/admin/users").status_code == 403
+
+    with app_conn(pg_settings) as conn:
+        run = conn.execute(
+            "INSERT INTO job_runs (job, trigger, finished_at, status, counts) VALUES ('integrity', 'schedule', now(),"
+            " 'degraded', %s) RETURNING id",
+            (Jsonb({"seal": 1, "asset_hash": 0}),),
+        ).fetchone()[0]
+        conn.execute(
+            "INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, 'seal', 'packet:7', %s)",
+            (run, Jsonb({"why": "hash differs"})),
+        )
+        conn.execute("INSERT INTO job_runs (job, trigger, status) VALUES ('integrity', 'schedule', 'running')")
+
+    sign_in(pg_client, "admin.ann")
+    alerts = pg_client.get("/api/admin/security/alerts", params={"hours": 1}).json()
+    kinds = {a["kind"] for a in alerts}
+    assert {"pin_failures", "stepup_wrong_pin", "denied", "integrity"} <= kinds, kinds
+    pins = next(a for a in alerts if a["kind"] == "pin_failures")
+    assert pins["display_name"] == "Andy Welder" and pins["detail"]["wrong_pins"] >= 3
+    steps = [a for a in alerts if a["kind"] == "stepup_wrong_pin"]
+    assert len(steps) == 3 and steps[0]["detail"]["where"] == "unlock"
+    denied = next(a for a in alerts if a["kind"] == "denied")
+    assert denied["display_name"] == "Clara Clerk" and denied["detail"]["path"] == "/api/admin/users"
+    assert [a["at"] for a in alerts] == sorted((a["at"] for a in alerts), reverse=True)
+
+    integ = pg_client.get("/api/admin/security/integrity").json()
+    assert integ["run"]["id"] == run and integ["run"]["status"] == "degraded"
+    assert integ["run"]["counts"] == {"seal": 1, "asset_hash": 0}
+    assert integ["findings"][0]["kind"] == "seal" and integ["findings"][0]["subject"] == "packet:7"
+    assert integ["findings"][0]["detail"] == {"why": "hash differs"}
+
+    who = pg_client.get("/api/admin/presence").json()
+    assert who["at"]
+    names = {s["display_name"]: s for s in who["sessions"]}
+    assert "Ann Admin" in names and names["Ann Admin"]["kind"] == "password" and names["Ann Admin"]["locked"] is False
+    assert "Andy Welder" not in names  # that session ended
+    with app_conn(pg_settings) as conn:
+        conn.execute(
+            "UPDATE sessions SET last_seen_at = now() - interval '20 minutes' WHERE user_id = %s",
+            (fake_dir.user("clerk.clara").guid,),
+        )
+    # Clara is a desk role (60 minutes), so she is still in; a shop worker idle 20 minutes is not
+    assert "Clara Clerk" in {s["display_name"] for s in pg_client.get("/api/admin/presence").json()["sessions"]}