Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/auth/directory.py

@@ -83,13 +83,43 @@ def get_user(conn: Any, user_id: Any) -> dict[str, Any] | None:
     return row
 
 
-def mark_inactive(conn: Any, user_id: Any, *, reason: str, gone: bool) -> None:
+def end_sessions(conn: Any, user_id: Any, reason: str, *, by: str | None = None) -> list[uuid.UUID]:
+    """End every live session of a user, one `auth.session_ended` audit row each. Returns the session ids."""
+    rows = conn.execute(
+        "SELECT id, roles_snapshot FROM sessions WHERE user_id = %s AND revoked_at IS NULL", (user_id,)
+    ).fetchall()
+    ended: list[uuid.UUID] = []
+    for sid, snap in rows:
+        if not sessions.revoke(conn, sid, reason):
+            continue
+        ended.append(sid)
+        detail: dict[str, Any] = {"reason": reason}
+        if by:
+            detail["by"] = by
+        audit.append(
+            conn,
+            action="auth.session_ended",
+            user_id=user_id,
+            session_id=sid,
+            role_snapshot=",".join((snap or {}).get("roles", [])),
+            detail=detail,
+        )
+    return ended
+
+
+def mark_inactive(conn: Any, user_id: Any, *, reason: str, gone: bool, audit_by: str | None = None) -> list[uuid.UUID]:
+    """Mark a user inactive (disabled or gone in AD), drop the roles and end the sessions.
+
+    With `audit_by` every ended session gets an `auth.session_ended` row naming who ended it. Returns the ids.
+    """
     conn.execute(
         "UPDATE users SET active = false, directory_flags = directory_flags || %s, ad_synced_at = now() WHERE id = %s",
         (Jsonb({"disabled": not gone, "gone": gone}), user_id),
     )
     conn.execute("DELETE FROM user_roles WHERE user_id = %s", (user_id,))
-    sessions.revoke_user(conn, user_id, reason)
+    if audit_by:
+        return end_sessions(conn, user_id, reason, by=audit_by)
+    return sessions.revoke_user(conn, user_id, reason)
 
 
 def public_user(row: dict[str, Any]) -> dict[str, Any]:
@@ -115,133 +145,195 @@ def ad_user_public(u: AdUser) -> dict[str, Any]:
     }
 
 
-# ---------------------------------------------------------------- AD refresh
+# ---------------------------------------------------------------- AD reconcile
+
+RECONCILE_BY = "ad-reconcile"
+LIST_CAP = 200  # names kept in a report list; the count is always whole
+
+
+class ReconcileRefused(Exception):
+    """AD answered, but with something that would deactivate people by mistake. Nothing was changed."""
 
 
 @dataclass
 class RefreshResult:
     groups_checked: int = 0
     groups_missing: list[str] = field(default_factory=list)
+    groups_extra: list[str] = field(default_factory=list)
     users_seen: int = 0
     created: int = 0
     updated: int = 0
     deactivated: int = 0
+    sessions_ended: int = 0
     emp_missing: list[str] = field(default_factory=list)
     emp_duplicate: list[str] = field(default_factory=list)
+    created_names: list[str] = field(default_factory=list)
+    deactivated_names: list[str] = field(default_factory=list)
 
     def as_dict(self) -> dict[str, Any]:
+        """The report: lists of names (missing and extra groups, accounts without an employee ID, duplicated
+        employee IDs) and counts. The same shape is the ad-reconcile run's counts."""
         return {
             "groups_checked": self.groups_checked,
-            "groups_missing": self.groups_missing,
+            "groups_missing": self.groups_missing[:LIST_CAP],
+            "groups_extra": self.groups_extra[:LIST_CAP],
             "users_seen": self.users_seen,
             "created": self.created,
             "updated": self.updated,
             "deactivated": self.deactivated,
-            "emp_missing": self.emp_missing,
-            "emp_duplicate": self.emp_duplicate,
+            "sessions_ended": self.sessions_ended,
+            "emp_missing": self.emp_missing[:LIST_CAP],
+            "emp_duplicate": self.emp_duplicate[:LIST_CAP],
+        }
+
+    def summary(self) -> dict[str, int]:
+        """Counts only, for the audit row and the log."""
+        return {
+            "groups_checked": self.groups_checked,
+            "groups_missing": len(self.groups_missing),
+            "groups_extra": len(self.groups_extra),
+            "users_seen": self.users_seen,
+            "created": self.created,
+            "updated": self.updated,
+            "deactivated": self.deactivated,
+            "sessions_ended": self.sessions_ended,
+            "emp_missing": len(self.emp_missing),
+            "emp_duplicate": len(self.emp_duplicate),
         }
 
 
-def refresh_from_ad(
-    conn: Any, directory: DirectoryProtocol, *, actor: sessions.Actor | None = None, ip: str | None = None
-) -> RefreshResult:
-    """Enumerate every mapped group, upsert its members and rebuild their roles; deactivate gone accounts.
+@dataclass
+class _AdRead:
+    """Everything the reconcile reads from AD, before any database write."""
 
-    All AD reads happen first, then the database writes in one transaction, so no lock is held while AD
-    answers. Raises LdapUnavailable when AD cannot be read.
-    """
-    res = RefreshResult()
+    present: list[tuple[str, str]]  # mapped groups found in the OU: (name as mapped, dn)
+    missing: list[str]
+    extra: list[str]
+    seen: dict[uuid.UUID, AdUser]
+    member_groups: dict[uuid.UUID, set[str]]
+    unseen: dict[uuid.UUID, AdUser | None]
+
+
+def _read_ad(conn: Any, directory: DirectoryProtocol) -> _AdRead:
     names = groupmap.group_names()
-    in_ad = {g.lower() for g in directory.list_groups()}
-    res.groups_missing = [n for n in names if n.lower() not in in_ad]
-    present = [n for n in names if n.lower() in in_ad]
+    in_ou = {cn.lower(): (cn, dn) for cn, dn in directory.list_groups_detail()}
+    mapped = {n.lower() for n in names}
+    present = [(n, in_ou[n.lower()][1]) for n in names if n.lower() in in_ou]
+    missing = [n for n in names if n.lower() not in in_ou]
+    extra = sorted((cn for key, (cn, _) in in_ou.items() if key not in mapped), key=str.lower)
+    if not present:
+        raise ReconcileRefused("no mapped group was found in the OU; check LDAP_GROUP_BASE_DN")
     seen: dict[uuid.UUID, AdUser] = {}
     member_groups: dict[uuid.UUID, set[str]] = {}
-    for name in present:
-        for u in directory.members_of(name):
+    for name, dn in present:
+        for u in directory.members_of_dn(dn):
             seen.setdefault(u.guid, u)
             member_groups.setdefault(u.guid, set()).add(name)
-    res.groups_checked = len(present)
-    res.users_seen = len(seen)
+    known = [r[0] for r in conn.execute("SELECT id FROM users WHERE active ORDER BY id").fetchall()]
+    unseen_ids = [uid for uid in known if uid not in seen]
+    if not seen and unseen_ids:
+        raise ReconcileRefused("AD listed no member in any mapped group; nobody was changed")
+    unseen = directory.users_by_guid(unseen_ids) if unseen_ids else {}
+    return _AdRead(present, missing, extra, seen, member_groups, unseen)
 
-    known = conn.execute("SELECT id FROM users WHERE active").fetchall()
-    unseen = [r[0] for r in known if r[0] not in seen]
-    # Accounts not in any group: still in AD (keep, no roles), disabled, or gone
-    unseen_state: dict[uuid.UUID, AdUser | None] = {uid: directory.get_user_by_guid(uid) for uid in unseen}
 
-    enabled = [u for u in seen.values() if not u.disabled]
+def refresh_from_ad(
+    conn: Any,
+    directory: DirectoryProtocol,
+    *,
+    actor: sessions.Actor | None = None,
+    ip: str | None = None,
+    run_id: int | None = None,
+) -> RefreshResult:
+    """The AD reconcile. List the OU (missing and extra groups), enumerate the members of every mapped group
+    (IN_CHAIN, paged), upsert them on objectGUID and rebuild their roles. An active user AD no longer lists in
+    any group is looked up by GUID: gone or disabled means inactive with the sessions ended (`ad_gone`,
+    `ad_disabled`); still there means no roles and the sessions ended (`roles_lost`). directory_flags gets
+    emp_missing, emp_duplicate, gone and disabled.
+
+    Every AD read happens first, then the writes in one transaction, so no lock is held while AD answers.
+    Raises LdapUnavailable when AD cannot be read and ReconcileRefused when what it read looks wrong.
+    """
+    res = RefreshResult()
+    ad = _read_ad(conn, directory)
+    res.groups_missing = ad.missing
+    res.groups_extra = ad.extra
+    res.groups_checked = len(ad.present)
+    res.users_seen = len(ad.seen)
+
+    enabled = [u for u in ad.seen.values() if not u.disabled]
+    enabled += [u for u in ad.unseen.values() if u is not None and not u.disabled]
     res.emp_missing = sorted(u.sam_account for u in enabled if not u.emp_no)
     counts = Counter(u.emp_no for u in enabled if u.emp_no)
     res.emp_duplicate = sorted(e for e, n in counts.items() if n > 1)
     dup = set(res.emp_duplicate)
+    who: dict[str, Any] = {
+        "user_id": actor.user_id if actor else None,
+        "session_id": actor.session_id if actor else None,
+        "role_snapshot": actor.role_snapshot if actor else None,
+    }
 
-    with conn.transaction():
-        for guid, u in seen.items():
-            grp = groupmap.mapped_groups(member_groups[guid])
-            was_active_row = conn.execute("SELECT active FROM users WHERE id = %s", (guid,)).fetchone()
-            created = upsert_user(conn, u, groups=grp)
-            conn.execute(
-                "UPDATE users SET directory_flags = directory_flags || %s WHERE id = %s",
-                (Jsonb({"emp_duplicate": bool(u.emp_no and u.emp_no in dup), "gone": False}), guid),
-            )
-            if created:
-                res.created += 1
-            else:
-                res.updated += 1
-            if u.disabled:
-                set_roles(conn, guid, [])
-                sessions.revoke_user(conn, guid, "ad_disabled")
-                if was_active_row is not None and was_active_row[0]:
-                    res.deactivated += 1
-                    audit.append(
-                        conn,
-                        action="ad.user_deactivated",
-                        user_id=actor.user_id if actor else None,
-                        session_id=actor.session_id if actor else None,
-                        role_snapshot=actor.role_snapshot if actor else None,
-                        subject_type="user",
-                        subject_id=guid,
-                        detail={"reason": "disabled", "sam_account": u.sam_account},
-                        ip=ip,
-                    )
-            else:
-                set_roles(conn, guid, groupmap.roles_from_groups(grp))
-        for uid, found in unseen_state.items():
-            if found is None or found.disabled:
-                mark_inactive(conn, uid, reason="ad_disabled" if found else "ad_gone", gone=found is None)
-                res.deactivated += 1
-                audit.append(
-                    conn,
-                    action="ad.user_deactivated",
-                    user_id=actor.user_id if actor else None,
-                    session_id=actor.session_id if actor else None,
-                    role_snapshot=actor.role_snapshot if actor else None,
-                    subject_type="user",
-                    subject_id=uid,
-                    detail={"reason": "gone" if found is None else "disabled"},
-                    ip=ip,
-                )
-            else:
-                upsert_user(conn, found, groups=[])
-                set_roles(conn, uid, [])
-                sessions.revoke_user(conn, uid, "roles_lost")
+    def deactivated(uid: uuid.UUID, reason: str, sam: str) -> None:
+        res.deactivated += 1
+        res.deactivated_names.append(sam)
         audit.append(
             conn,
-            action="ad.refresh",
-            user_id=actor.user_id if actor else None,
-            session_id=actor.session_id if actor else None,
-            role_snapshot=actor.role_snapshot if actor else None,
-            detail={
-                "groups_checked": res.groups_checked,
-                "groups_missing": len(res.groups_missing),
-                "users_seen": res.users_seen,
-                "created": res.created,
-                "updated": res.updated,
-                "deactivated": res.deactivated,
-                "emp_missing": len(res.emp_missing),
-                "emp_duplicate": len(res.emp_duplicate),
-            },
+            action="ad.user_deactivated",
+            subject_type="user",
+            subject_id=uid,
+            detail={"reason": reason, "sam_account": sam},
             ip=ip,
+            **who,
         )
-    log.info("AD refresh: %s", {k: v for k, v in res.as_dict().items() if isinstance(v, int)})
+
+    def flags(uid: uuid.UUID, u: AdUser | None) -> None:
+        conn.execute(
+            "UPDATE users SET directory_flags = directory_flags || %s WHERE id = %s",
+            (Jsonb({"emp_duplicate": bool(u and u.emp_no and u.emp_no in dup), "gone": u is None}), uid),
+        )
+
+    with conn.transaction():
+        for guid, u in ad.seen.items():
+            grp = groupmap.mapped_groups(ad.member_groups[guid])
+            was = conn.execute("SELECT active FROM users WHERE id = %s", (guid,)).fetchone()
+            if upsert_user(conn, u, groups=grp):
+                res.created += 1
+                res.created_names.append(u.sam_account)
+            else:
+                res.updated += 1
+            flags(guid, u)
+            roles = [] if u.disabled else groupmap.roles_from_groups(grp)
+            set_roles(conn, guid, roles)
+            if u.disabled:
+                res.sessions_ended += len(end_sessions(conn, guid, "ad_disabled", by=RECONCILE_BY))
+                if was is not None and was[0]:
+                    deactivated(guid, "disabled", u.sam_account)
+            elif not roles:
+                res.sessions_ended += len(end_sessions(conn, guid, "roles_lost", by=RECONCILE_BY))
+        for uid, found in ad.unseen.items():
+            if found is None or found.disabled:
+                row = conn.execute("SELECT sam_account FROM users WHERE id = %s", (uid,)).fetchone()
+                ended = mark_inactive(
+                    conn,
+                    uid,
+                    reason="ad_gone" if found is None else "ad_disabled",
+                    gone=found is None,
+                    audit_by=RECONCILE_BY,
+                )
+                res.sessions_ended += len(ended)
+                flags(uid, found)
+                deactivated(uid, "gone" if found is None else "disabled", row[0] if row else str(uid))
+            else:
+                upsert_user(conn, found, groups=[])
+                flags(uid, found)
+                set_roles(conn, uid, [])
+                res.sessions_ended += len(end_sessions(conn, uid, "roles_lost", by=RECONCILE_BY))
+        detail: dict[str, Any] = dict(res.summary())
+        if run_id is not None:
+            detail["run_id"] = run_id
+        audit.append(conn, action="ad.refresh", detail=detail, ip=ip, **who)
+    log.info("AD reconcile: %s", res.summary())
     return res
+
+
+reconcile = refresh_from_ad