Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/server/test_admin_api.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from fakedir import FAKE_PASSWORD, FakeDirectory
 from fastapi.testclient import TestClient
 from pshelpers import ORIGIN_HEADERS, PIN, app_conn, needs_pg, pin_signin, register_device, set_pin, sign_in
+from psycopg.types.json import Jsonb
 
 from packetstation.main import create_app
 from packetstation.settings import Settings
@@ -34,53 +35,46 @@ def test_group_map_flags_missing_and_extra(pg_client: TestClient, fake_dir: Fake
     assert weld["role"] == "shop:03:lead" and weld["dept_code"] == "03"
 
 
-def test_ad_refresh_counts_and_flags(pg_client: TestClient, pg_settings: Settings, fake_dir: FakeDirectory) -> None:
+def test_ad_refresh_requests_a_reconcile_run(pg_client: TestClient, pg_settings: Settings) -> None:
+    """POST /ad/refresh asks the worker for an ad-reconcile run; the counts are the reconcile's (test_reconcile)."""
     sign_in(pg_client, "admin.ann")
-    res = pg_client.post("/api/admin/ad/refresh").json()
-    assert res == {
-        "groups_checked": 49,
-        "groups_missing": ["PS-14-Electrical-Lead"],
-        "users_seen": 16,
-        "created": 15,
-        "updated": 1,
+    r = pg_client.post("/api/admin/ad/refresh")
+    assert r.status_code == 202, r.text
+    assert r.json() == {"requested": True, "pending": 1, "job": "ad-reconcile"}
+    assert pg_client.post("/api/admin/ad/refresh").json()["pending"] == 2
+    empty = pg_client.get("/api/admin/ad/reconcile").json()
+    assert empty["run"] is None and empty["pending"] == 2
+    assert empty["report"] == {
+        "groups_missing": [],
+        "groups_extra": [],
+        "users_seen": 0,
+        "created": 0,
+        "updated": 0,
         "deactivated": 0,
-        "emp_missing": ["noemp.nora"],
-        "emp_duplicate": ["9001"],
+        "sessions_ended": 0,
+        "emp_missing": [],
+        "emp_duplicate": [],
     }
-    users = {u["sam_account"]: u for u in pg_client.get("/api/admin/users").json()}
-    assert "norole.nick" not in users and len(users) == 16
-    assert users["dup.dave"]["flags"]["emp_duplicate"] and users["dup.dina"]["flags"]["emp_duplicate"]
-    assert users["noemp.nora"]["flags"]["emp_missing"] and users["noemp.nora"]["emp_no"] is None
-    assert users["disabled.dan"]["active"] is False and users["disabled.dan"]["flags"]["disabled"]
-    assert users["nested.ned"]["roles"] == ["shop:10:worker"]
-    assert users["both.bo"]["roles"] == ["shop:03:supervisor"]
-    assert users["multi.mia"]["roles"] == ["clerical", "shop:06:supervisor"]
-    assert [u["sam_account"] for u in pg_client.get("/api/admin/users", params={"q": "dup"}).json()] == [
-        "dup.dave",
-        "dup.dina",
-    ]
-
-    # Clara signs in elsewhere, then AD disables her; the next refresh deactivates her and ends the session
-
-    with TestClient(create_app(pg_settings, directory=fake_dir), headers=ORIGIN_HEADERS) as other:
-        sign_in(other, "clerk.clara")
-        fake_dir.user("clerk.clara").disabled = True
-        fake_dir.users.remove(fake_dir.user("wh.wade"))  # gone from AD
-        fake_dir.user("qc.quinn").groups = []  # still in AD, in no group
-        res = pg_client.post("/api/admin/ad/refresh").json()
-        assert res["deactivated"] == 2 and res["users_seen"] == 14
-        assert other.get("/api/auth/me").json()["code"] == "no_session"
-    users = {u["sam_account"]: u for u in pg_client.get("/api/admin/users").json()}
-    assert users["clerk.clara"]["active"] is False and users["wh.wade"]["active"] is False
-    assert users["qc.quinn"]["active"] is True and users["qc.quinn"]["roles"] == []
     with app_conn(pg_settings) as conn:
+        assert conn.execute("SELECT count(*) FROM job_requests WHERE job = 'ad-reconcile'").fetchone()[0] == 2
         acts = conn.execute("SELECT action FROM audit_events WHERE action LIKE 'ad.%' ORDER BY id").fetchall()
-        assert [a[0] for a in acts] == ["ad.refresh", "ad.user_deactivated", "ad.user_deactivated", "ad.refresh"]
+        assert [a[0] for a in acts] == ["ad.refresh_requested", "ad.refresh_requested"]
+        conn.execute(
+            "INSERT INTO job_runs (job, trigger, finished_at, status, counts) VALUES ('ad-reconcile', 'request',"
+            " now(), 'ok', %s)",
+            (Jsonb({"users_seen": 16, "created": 15, "groups_missing": ["PS-14-Electrical-Lead"], "x": 1}),),
+        )
+        conn.execute("INSERT INTO job_runs (job, trigger, status) VALUES ('ad-reconcile', 'schedule', 'running')")
+    got = pg_client.get("/api/admin/ad/reconcile").json()
+    assert got["run"]["status"] == "ok" and got["run"]["trigger"] == "request"
+    assert got["report"]["users_seen"] == 16 and got["report"]["created"] == 15
+    assert got["report"]["groups_missing"] == ["PS-14-Electrical-Lead"] and "x" not in got["report"]
 
 
 def test_user_detail_and_lookup(pg_client: TestClient, fake_dir: FakeDirectory) -> None:
+    sign_in(pg_client, "multi.mia")  # the user row exists
+    pg_client.post("/api/auth/signout")
     sign_in(pg_client, "admin.ann")
-    pg_client.post("/api/admin/ad/refresh")
     mia = next(u for u in pg_client.get("/api/admin/users", params={"q": "mia"}).json())
     detail = pg_client.get(f"/api/admin/users/{mia['id']}").json()
     assert detail["groups"] == ["PS-06-CodeBldg2-Supervisor", "PS-Clerical"] and detail["groups_source"] == "ad"