Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/server/test_reconcile.py

@@ -0,0 +1,217 @@
+"""The ad-reconcile job against the fake directory: the report, users, flags, ended sessions and events."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import date, datetime
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import pytest
+from fakedir import BASE_DN, NESTED_GROUP, OU_DN, FakeDirectory
+from pshelpers import app_conn, needs_pg
+
+from packetstation.auth import directory as userdir
+from packetstation.auth import groups, sessions
+from packetstation.feed.config import FeedSettings
+from packetstation.settings import Settings
+from packetstation.worker import jobs, runs
+from packetstation.worker.jobs import ad_reconcile, load_jobs
+from packetstation.worker.scheduler import Scheduler
+
+TZ = ZoneInfo("America/Denver")
+
+
+class Events:
+    """Records raise_event and resolve_event calls in place of the notify package."""
+
+    def __init__(self) -> None:
+        self.raised: list[tuple[str, dict[str, Any]]] = []
+        self.resolved: list[tuple[str, dict[str, Any]]] = []
+
+    def raise_event(self, conn: Any, event_type: str, **kw: Any) -> int:
+        self.raised.append((event_type, kw))
+        return len(self.raised)
+
+    def resolve_event(self, conn: Any, event_type: str, **kw: Any) -> None:
+        self.resolved.append((event_type, kw))
+
+    def of(self, event_type: str) -> list[dict[str, Any]]:
+        return [kw for t, kw in self.raised if t == event_type]
+
+
+@pytest.fixture
+def events(monkeypatch: pytest.MonkeyPatch) -> Events:
+    ev = Events()
+    monkeypatch.setattr(jobs, "raise_event", ev.raise_event)
+    monkeypatch.setattr(jobs, "resolve_event", ev.resolve_event)
+    return ev
+
+
+def scheduler(settings: Settings, d: FakeDirectory) -> Scheduler:
+    sch = Scheduler(settings, build="pytest", jobs={"ad-reconcile": ad_reconcile.SPEC})
+    sch.stack.cache["directory"] = d
+    return sch
+
+
+def reconcile(settings: Settings, d: FakeDirectory, trigger: str = "schedule") -> dict[str, Any]:
+    return runs.run_job(scheduler(settings, d), "ad-reconcile", trigger)
+
+
+def live_session(settings: Settings, conn: Any, user_id: uuid.UUID) -> uuid.UUID:
+    _, row = sessions.create(
+        conn, settings, settings.session_key(), user_id=user_id, kind="password", roles=["clerical"], groups=[]
+    )
+    sid: uuid.UUID = row["id"]
+    return sid
+
+
+# ---------------------------------------------------------------- no database
+
+
+def test_registered_with_cadence() -> None:
+    reg = load_jobs()
+    for name in ("ad-reconcile", "session-refresh", "integrity", "backup-check"):
+        assert name in reg and reg[name].pausable is False, name
+    fs = FeedSettings()
+    at = datetime(2026, 10, 21, 1, 0, tzinfo=TZ)
+    assert reg["ad-reconcile"].cadence(fs, at) == 30 * 60
+    assert reg["integrity"].cadence(fs, at) == 3 * 3600
+    assert reg["backup-check"].cadence(fs, at) == 6 * 3600 + 15 * 60
+
+
+def test_fake_directory_additions() -> None:
+    d = FakeDirectory()
+    detail = d.list_groups_detail()
+    names = [n for n, _ in detail]
+    assert "PS-Extra-Legacy" in names and "PS-14-Electrical-Lead" not in names
+    assert dict(detail)["PS-Admins"] == f"CN=PS-Admins,{OU_DN}"
+    weld10 = {u.sam_account for u in d.members_of_dn(dict(detail)["PS-10-WeldAssembly-Worker"])}
+    assert weld10 == {"nested.ned"}
+    weld03 = {u.sam_account for u in d.members_of_dn(dict(detail)["PS-03-WeldParts-Worker"])}
+    assert "disabled.dan" in weld03  # disabled accounts are listed too
+    assert d.members_of_dn(f"CN=Nope,{OU_DN}") == []
+    found = d.search_groups("helpdesk")
+    helpdesk = {"dn": f"CN=IT-Helpdesk,OU=Groups,OU=PESCO_FARMINGTON,{BASE_DN}", "name": "IT-Helpdesk"}
+    assert found == [{**helpdesk, "description": "IT help desk"}]
+    assert [g["name"] for g in d.search_groups("nested")] == [NESTED_GROUP]
+    assert len(d.search_groups("PS-", limit=5)) == 5
+    ann, ghost = d.user("admin.ann").guid, uuid.uuid4()
+    got = d.users_by_guid([ann, ghost])
+    assert got[ghost] is None and got[ann] is not None and got[ann].sam_account == "admin.ann"
+
+
+def test_report_text() -> None:
+    res = userdir.RefreshResult(created=2, created_names=["a", "b"], groups_extra=["X"], emp_duplicate=["9001"])
+    text = ad_reconcile.report_text(res)
+    assert text is not None
+    assert text[0] == "AD reconcile: 2 added, 1 extra group, 1 duplicate employee ID"
+    assert "Added: a, b." in text[1] and "9001" in text[1]
+    assert ad_reconcile.report_text(userdir.RefreshResult(updated=5)) is None
+
+
+# ---------------------------------------------------------------- PostgreSQL
+
+
+@needs_pg
+def test_reconcile_run(pg_settings: Settings, events: Events) -> None:
+    d = FakeDirectory()
+    row = reconcile(pg_settings, d)
+    assert row["status"] == "ok", row["errors"]
+    c = row["counts"]
+    assert c["groups_missing"] == ["PS-14-Electrical-Lead"] and c["groups_extra"] == ["PS-Extra-Legacy"]
+    assert c["groups_checked"] == 49 and c["users_seen"] == 16 and c["created"] == 16 and c["deactivated"] == 0
+    assert c["emp_missing"] == ["noemp.nora"] and c["emp_duplicate"] == ["9001"] and c["sessions_ended"] == 0
+    assert [kw["subject"] for kw in events.of("group_missing")] == ["PS-14-Electrical-Lead"]
+    assert events.of("group_missing")[0]["condition"] is True
+    resolved = {kw["subject"] for t, kw in events.resolved if t == "group_missing"}
+    assert "PS-Admins" in resolved and "PS-14-Electrical-Lead" not in resolved
+    (report,) = events.of("ad_reconcile_report")
+    assert report["title"].startswith("AD reconcile: 16 added") and "PS-Extra-Legacy" in report["body"]
+
+    with app_conn(pg_settings) as conn:
+        users = {
+            r[0]: (r[1], r[2])
+            for r in conn.execute("SELECT sam_account, active, directory_flags FROM users").fetchall()
+        }
+        assert "norole.nick" not in users
+        assert users["disabled.dan"][0] is False and users["disabled.dan"][1]["disabled"] is True
+        assert users["dup.dave"][1]["emp_duplicate"] and users["dup.dina"][1]["emp_duplicate"]
+        assert users["noemp.nora"][1]["emp_missing"] and not users["clerk.clara"][1]["emp_duplicate"]
+        assert users["clerk.clara"][1]["gone"] is False
+        ned = d.user("nested.ned").guid
+        assert userdir.roles_of(conn, ned) == ["shop:10:worker"]
+
+        # Sessions for the people about to change
+        clara, wade, quinn, ann = (d.user(s).guid for s in ("clerk.clara", "wh.wade", "qc.quinn", "admin.ann"))
+        sid = {u: live_session(pg_settings, conn, u) for u in (clara, wade, quinn, ann)}
+
+    d.user("clerk.clara").disabled = True  # disabled in AD, still in her group
+    d.users.remove(d.user("wh.wade"))  # gone from AD
+    d.user("qc.quinn").groups = []  # still in AD, no group
+    d.ou_groups.append("PS-14-Electrical-Lead")  # IT created the missing group
+    events.raised.clear()
+    events.resolved.clear()
+    with app_conn(pg_settings) as conn:
+        conn.execute("INSERT INTO job_requests (job, requested_by) VALUES ('ad-reconcile', %s)", (ann,))
+    row = reconcile(pg_settings, d)
+    assert row["status"] == "ok" and row["trigger"] == "request" and row["requested_by"] == [ann]
+    c = row["counts"]
+    assert c["deactivated"] == 2 and c["sessions_ended"] == 3 and c["users_seen"] == 14 and c["created"] == 0
+    assert c["groups_missing"] == []
+    assert events.of("group_missing") == []
+    elec = "PS-14-Electrical-Lead"
+    assert ("group_missing", {"subject": elec, "title": f"Group missing from the OU: {elec}"}) in events.resolved
+    (report,) = events.of("ad_reconcile_report")
+    assert "2 deactivated" in report["title"] and "clerk.clara" in report["body"] and "wh.wade" in report["body"]
+
+    with app_conn(pg_settings) as conn:
+        ended = dict(conn.execute("SELECT id, revoked_reason FROM sessions").fetchall())
+        assert ended[sid[clara]] == "ad_disabled" and ended[sid[wade]] == "ad_gone"
+        assert ended[sid[quinn]] == "roles_lost" and ended[sid[ann]] is None
+        state = {r[0]: (r[1], r[2]) for r in conn.execute("SELECT id, active, directory_flags FROM users")}
+        assert state[wade][0] is False and state[wade][1]["gone"] is True
+        assert state[clara][0] is False and state[clara][1]["disabled"] is True
+        assert state[quinn][0] is True and userdir.roles_of(conn, quinn) == []
+        acts = conn.execute(
+            "SELECT action, detail->>'reason', detail->>'by' FROM audit_events"
+            " WHERE action IN ('auth.session_ended', 'ad.user_deactivated', 'ad.refresh') ORDER BY id"
+        ).fetchall()
+        assert [a[0] for a in acts].count("ad.refresh") == 2
+        session_ends = sorted((a[1], a[2]) for a in acts if a[0] == "auth.session_ended")
+        assert session_ends == [
+            ("ad_disabled", "ad-reconcile"),
+            ("ad_gone", "ad-reconcile"),
+            ("roles_lost", "ad-reconcile"),
+        ]
+
+
+@needs_pg
+def test_reconcile_down_or_empty_changes_nothing(pg_settings: Settings, events: Events) -> None:
+    d = FakeDirectory()
+    assert reconcile(pg_settings, d)["status"] == "ok"
+    d.down = True
+    row = reconcile(pg_settings, d)
+    assert row["status"] == "failed" and "LdapUnavailable" in row["errors"][0]
+    d.down = False
+    d.ou_groups = ["PS-Extra-Legacy"]  # a wrong group base DN: no mapped group at all
+    row = reconcile(pg_settings, d)
+    assert row["status"] == "failed" and "ReconcileRefused" in row["errors"][0]
+    d.ou_groups = list(groups.group_names())
+    d.users = []  # every group empty
+    row = reconcile(pg_settings, d)
+    assert row["status"] == "failed" and "ReconcileRefused" in row["errors"][0]
+    # Each stopped run raises the one "did not run" condition (the notify dedupe keeps it to one mail)
+    stopped = [kw for kw in events.of("ad_reconcile_report") if kw["subject"] == ad_reconcile.FAILED_SUBJECT]
+    assert len(stopped) == 3 and all(kw["condition"] is True for kw in stopped)
+    assert stopped[0]["title"] == "AD reconcile did not run" and "LdapUnavailable" in stopped[0]["body"]
+    with app_conn(pg_settings) as conn:
+        assert conn.execute("SELECT count(*) FROM users WHERE active").fetchone()[0] == 15
+        assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'ad.refresh'").fetchone()[0] == 1
+
+
+def test_cadence_is_site_time() -> None:
+    fs = FeedSettings()
+    day = date(2026, 10, 21)
+    assert ad_reconcile.cadence(fs, datetime(day.year, day.month, day.day, 1, 29, tzinfo=TZ)) == 60
+    assert ad_reconcile.cadence(fs, datetime(day.year, day.month, day.day, 1, 30, tzinfo=TZ)) == 24 * 3600