Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/api/admin.py

@@ -22,10 +22,12 @@ from ..auth import groups as groupmap
 from ..auth import pin as pins
 from ..auth.ldap import LdapUnavailable
 from ..auth.signin import ldap_down
-from ..domain import assets, scan
+from ..domain import assets, profiles, scan
 from .account import active_sessions
 from .deps import Db, Principal, admin_principal, directory_of, get_db, settings_of
 from .errors import ApiError
+from .jobs import request_run, run_out
+from .packets import WRITE
 
 router = APIRouter(prefix="/api/admin", tags=["admin"])
 
@@ -61,7 +63,7 @@ USER_SELECT = (
 )
 
 
-def _user_row(r: dict[str, Any], roles: list[str]) -> dict[str, Any]:
+def _user_row(r: dict[str, Any], roles: list[str], limits: dict[str, list[int]] | None = None) -> dict[str, Any]:
     flags = r.get("directory_flags") or {}
     return {
         "id": str(r["id"]),
@@ -79,6 +81,10 @@ def _user_row(r: dict[str, Any], roles: list[str]) -> dict[str, Any]:
         },
         "last_signin_at": r["last_signin_at"],
         "ad_synced_at": r["ad_synced_at"],
+        "welder_stamp": r.get("welder_stamp"),
+        "profile_active": bool(r.get("profile_active", True)),
+        "station_limits": limits or {},
+        "directory_flags": {k: v for k, v in flags.items() if k != "groups"},
     }
 
 
@@ -100,8 +106,10 @@ def users(
             ).fetchall()
         else:
             rows = cur.execute(USER_SELECT + " ORDER BY lower(u.display_name) LIMIT 500").fetchall()
-    roles = _roles_by_user(conn, [r["id"] for r in rows])
-    return [_user_row(r, roles.get(r["id"], [])) for r in rows]
+    ids = [r["id"] for r in rows]
+    roles = _roles_by_user(conn, ids)
+    limits = profiles.station_limits(conn, ids)
+    return [_user_row(r, roles.get(r["id"], []), limits.get(r["id"])) for r in rows]
 
 
 @router.get("/users/{user_id}")
@@ -116,7 +124,9 @@ def user_detail(
         r = cur.execute(USER_SELECT + " WHERE u.id = %s", (user_id,)).fetchone()
     if r is None:
         raise ApiError(404, "not_found", "No such user.")
-    out = _user_row(r, _roles_by_user(conn, [user_id]).get(user_id, []))
+    out = _user_row(
+        r, _roles_by_user(conn, [user_id]).get(user_id, []), profiles.station_limits(conn, [user_id]).get(user_id)
+    )
     directory = directory_of(request)
     try:
         u = directory.get_user_by_guid(user_id)
@@ -238,7 +248,7 @@ def revoke_device(
     return Response(status_code=204)
 
 
-def _group_map_view(request: Request) -> dict[str, Any]:
+def _groups_cache(request: Request) -> dict[str, Any]:
     cache: dict[str, Any] | None = getattr(request.app.state, "groups_cache", None)
     if cache is None or time.monotonic() - cache["t"] > GROUPS_CACHE_SECONDS:
         try:
@@ -247,6 +257,17 @@ def _group_map_view(request: Request) -> dict[str, Any]:
             in_ad = None
         cache = {"t": time.monotonic(), "checked_at": datetime.now(UTC), "in_ad": in_ad}
         request.app.state.groups_cache = cache
+    return cache
+
+
+def groups_in_ou(request: Request) -> list[str] | None:
+    """The group names in the PacketStation OU (cached GROUPS_CACHE_SECONDS), or None when AD is not reachable."""
+    names: list[str] | None = _groups_cache(request)["in_ad"]
+    return names
+
+
+def _group_map_view(request: Request) -> dict[str, Any]:
+    cache = _groups_cache(request)
     names: list[str] | None = cache["in_ad"]
     lower = {n.lower() for n in names} if names is not None else None
     gm = groupmap.current()
@@ -269,18 +290,46 @@ def group_map(request: Request, p: Principal = Depends(admin_principal)) -> dict
     return _group_map_view(request)
 
 
-@router.post("/ad/refresh")
+AD_RECONCILE_JOB = "ad-reconcile"
+REPORT_KEYS: dict[str, Any] = {
+    "groups_missing": list,
+    "groups_extra": list,
+    "users_seen": int,
+    "created": int,
+    "updated": int,
+    "deactivated": int,
+    "sessions_ended": int,
+    "emp_missing": list,
+    "emp_duplicate": list,
+}
+
+
+@router.post("/ad/refresh", status_code=202, dependencies=WRITE)
 def ad_refresh(
     request: Request,
     p: Principal = Depends(admin_principal),  # noqa: B008
     d: Db = Depends(get_db),  # noqa: B008
 ) -> dict[str, Any]:
-    try:
-        res = userdir.refresh_from_ad(d.conn, directory_of(request), actor=p.actor, ip=p.ip)
-    except LdapUnavailable as e:
-        raise ldap_down(e) from None
+    """Ask the worker for an ad-reconcile run now. The result shows on GET /ad/reconcile."""
+    res = request_run(d.conn, p, AD_RECONCILE_JOB, action="ad.refresh_requested")
     request.app.state.groups_cache = None
-    return res.as_dict()
+    return {**res, "job": AD_RECONCILE_JOB}
+
+
+@router.get("/ad/reconcile")
+def ad_reconcile(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    """The newest finished ad-reconcile run and its report."""
+    with d.conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT * FROM job_runs WHERE job = %s AND finished_at IS NOT NULL ORDER BY id DESC LIMIT 1",
+            (AD_RECONCILE_JOB,),
+        ).fetchone()
+        pending = cur.execute(
+            "SELECT count(*) AS n FROM job_requests WHERE job = %s AND claimed_at IS NULL", (AD_RECONCILE_JOB,)
+        ).fetchone()
+    counts = (r or {}).get("counts") or {}
+    report = {k: counts.get(k, kind()) for k, kind in REPORT_KEYS.items()}
+    return {"run": run_out(r), "report": report, "pending": int(pending["n"]) if pending else 0}
 
 
 @router.get("/signin-attempts")

server/packetstation/api/admin_org.py

@@ -0,0 +1,287 @@
+"""Admin: departments and stations, people's profiles (one at a time and the CSV import), checklist templates.
+
+Every route needs the admin role and a password session. Every change is audited (`admin.*`).
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.encoders import jsonable_encoder
+from psycopg.types.json import Jsonb
+from pydantic import BaseModel, ConfigDict, Field
+
+from .. import audit
+from ..auth import groups as groupmap
+from ..domain import checklists, profiles
+from ..domain import departments as depts
+from .admin import groups_in_ou
+from .deps import Db, Principal, admin_principal, get_db
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+IMPORT_MAX_BODY = profiles.CSV_MAX_BYTES * 2 + 10_000  # JSON escaping can double the text
+
+
+def import_limit(request: Request) -> None:
+    length = request.headers.get("content-length")
+    if length is not None and length.strip().isdigit() and int(length) > IMPORT_MAX_BODY:
+        raise ApiError(413, "too_large", "The file is larger than 1 MB.")
+
+
+def reload_groups(request: Request, conn: Any) -> None:
+    """Rebuild the group map from the departments table and drop the cached OU listing."""
+    found = groupmap.load_departments(conn)
+    if found:
+        groupmap.use_departments(found)
+    request.app.state.groups_cache = None
+
+
+# ---------------------------------------------------------------- departments
+
+
+class StationIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: int | None = None
+    name: str = Field(max_length=200)
+    sort: int = Field(0, ge=0, le=100_000)
+    active: bool = True
+
+
+class DepartmentPatch(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str | None = Field(None, max_length=200)
+    active: bool | None = None
+    stations: list[StationIn] | None = Field(None, max_length=depts.STATIONS_MAX)
+
+
+class DepartmentNew(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    code: str = Field(max_length=10)
+    slug: str = Field(max_length=40)
+    name: str = Field(max_length=200)
+    kind: str = Field(pattern="^(shop|support)$")
+    weld: bool = False
+    supervisor_group: bool = True
+    stations: list[str] = Field(default_factory=list, max_length=depts.STATIONS_MAX)
+
+
+@router.get("/departments")
+def department_list(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> list[dict[str, Any]]:
+    return depts.list_all(d.conn, groups_in_ou(request))
+
+
+@router.patch("/departments/{code}", dependencies=WRITE)
+def department_update(
+    code: str,
+    body: DepartmentPatch,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    depts.update(
+        conn,
+        p,
+        code[:10],
+        name=body.name,
+        active=body.active,
+        stations=[s.model_dump() for s in body.stations] if body.stations is not None else None,
+    )
+    reload_groups(request, conn)
+    out = depts.get(conn, code[:10], groups_in_ou(request))
+    assert out is not None
+    return out
+
+
+@router.post("/departments", status_code=201, dependencies=WRITE)
+def department_add(
+    body: DepartmentNew,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    _, groups = depts.add(
+        conn,
+        p,
+        code=body.code,
+        slug=body.slug,
+        name=body.name,
+        kind=body.kind,
+        weld=body.weld,
+        supervisor_group=body.supervisor_group,
+        stations=body.stations,
+    )
+    reload_groups(request, conn)
+    dept = depts.get(conn, body.code.strip(), groups_in_ou(request))
+    return {"department": dept, "groups_to_create": groups}
+
+
+# ---------------------------------------------------------------- profiles
+
+
+class ProfilePatch(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    welder_stamp: str | None = Field(None, max_length=40)
+    profile_active: bool | None = None
+    station_limits: dict[str, list[int]] | None = None
+
+
+class ImportBody(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    csv: str
+    apply: bool = False
+
+
+@router.patch("/users/{user_id}/profile", dependencies=WRITE)
+def profile_update(
+    user_id: uuid.UUID,
+    body: ProfilePatch,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    fields = body.model_dump(exclude_unset=True)
+    out = profiles.update(d.conn, p, user_id, fields)
+    return {"user_id": str(user_id), **out}
+
+
+@router.post("/users/import", dependencies=[Depends(import_limit)])
+def profile_import(
+    body: ImportBody,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    return profiles.import_profiles(d.conn, p, body.csv, apply=body.apply)
+
+
+# ---------------------------------------------------------------- checklists
+
+RX_BRACES = re.compile(r"\{[^{}]*\}")
+ALLOWED_PLACEHOLDERS = ("{drawing}", "{rev}", "{wps}")
+
+
+class ChecklistsPut(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    value: dict[str, Any]
+    version: int = Field(ge=0)
+
+
+def check_templates(value: Any) -> tuple[dict[str, list[dict[str, str]]], list[str]]:
+    """The checklist templates cleaned, and one error line per problem."""
+    errors: list[str] = []
+    out: dict[str, list[dict[str, str]]] = {}
+    if not isinstance(value, dict):
+        return {}, ["value must be an object with weld and plain lists"]
+    extra = sorted(k for k in value if k not in ("weld", "plain"))
+    if extra:
+        errors.append(f"unknown list: {', '.join(extra)}")
+    for name in ("weld", "plain"):
+        items = value.get(name)
+        if not isinstance(items, list) or not items:
+            errors.append(f"{name}: at least one item")
+            continue
+        if len(items) > checklists.ITEMS_MAX:
+            errors.append(f"{name}: at most {checklists.ITEMS_MAX} items")
+        seen: set[str] = set()
+        clean: list[dict[str, str]] = []
+        for i, it in enumerate(items, start=1):
+            where = f"{name} item {i}"
+            if not isinstance(it, dict):
+                errors.append(f"{where}: must be an object with group, key and text")
+                continue
+            group = " ".join(str(it.get("group") or "").split())
+            key = str(it.get("key") or "").strip()
+            text = " ".join(str(it.get("text") or "").split())
+            if not checklists.RX_KEY.match(key):
+                errors.append(f"{where}: the key is a lower-case letter, then letters, digits or _ (at most 30)")
+            elif key in seen:
+                errors.append(f"{where}: key {key} is used twice")
+            seen.add(key)
+            if not group:
+                errors.append(f"{where}: the group is empty")
+            if not text:
+                errors.append(f"{where}: the text is empty")
+            if len(group) > checklists.TEXT_MAX or len(text) > checklists.TEXT_MAX:
+                errors.append(f"{where}: at most {checklists.TEXT_MAX} characters")
+            bad = [m for m in RX_BRACES.findall(text) if m not in ALLOWED_PLACEHOLDERS]
+            stray = RX_BRACES.sub("", text)
+            if bad:
+                errors.append(f"{where}: unknown placeholder {', '.join(bad)} (use {{drawing}}, {{rev}} or {{wps}})")
+            elif "{" in stray or "}" in stray:
+                errors.append(f"{where}: a brace without its pair")
+            clean.append({"group": group, "key": key, "text": text})
+        out[name] = clean
+    return out, errors
+
+
+def _checklists_out(conn: Any) -> dict[str, Any]:
+    row = conn.execute("SELECT version FROM settings WHERE key = %s", (checklists.SETTINGS_KEY,)).fetchone()
+    return {
+        "value": checklists.load_templates(conn),
+        "defaults": checklists.DEFAULT_TEMPLATES,
+        "version": int(row[0]) if row else 0,
+    }
+
+
+@router.get("/checklists")
+def checklists_get(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    return _checklists_out(d.conn)
+
+
+@router.put("/checklists", dependencies=WRITE)
+def checklists_put(
+    body: ChecklistsPut,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    value, errors = check_templates(body.value)
+    if errors:
+        raise ApiError(422, "bad_checklists", errors[0], extra={"errors": errors})
+    with conn.transaction():
+        row = conn.execute(
+            "SELECT version FROM settings WHERE key = %s FOR UPDATE", (checklists.SETTINGS_KEY,)
+        ).fetchone()
+        current = int(row[0]) if row else 0
+        if body.version != current:
+            raise ApiError(
+                409,
+                "version_conflict",
+                "Someone else changed the checklists. Reload and try again.",
+                extra={"current": jsonable_encoder(_checklists_out(conn))},
+            )
+        before = checklists.load_templates(conn)
+        conn.execute(
+            "INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
+            " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
+            " updated_at = now(), updated_by = EXCLUDED.updated_by",
+            (checklists.SETTINGS_KEY, Jsonb(value), p.user_id),
+        )
+        audit.append(
+            conn,
+            action="admin.checklists_updated",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="settings",
+            subject_id=checklists.SETTINGS_KEY,
+            detail={"before": before, "after": value},
+            ip=p.ip,
+        )
+    return _checklists_out(conn)

server/packetstation/api/admin_security.py

@@ -0,0 +1,634 @@
+"""Admin: the Security tab (settings, alerts, the integrity result), the access log and its CSV, who is in now, the
+stamp check, and the directory flags.
+
+Every route needs the admin role and a password session. Every change is audited (`admin.*`).
+"""
+
+from __future__ import annotations
+
+import io
+import ipaddress
+import json
+import re
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query, Request, Response
+from fastapi.encoders import jsonable_encoder
+from psycopg.rows import dict_row
+from pydantic import BaseModel, ConfigDict, Field
+from starlette.concurrency import run_in_threadpool
+from starlette.datastructures import UploadFile
+
+from .. import audit
+from ..auth import secsettings
+from ..domain import security_alerts
+from ..pdf import stamps, tracecsv
+from .account import LOG_PREFIXES
+from .deps import Db, Principal, admin_principal, get_db, idle_minutes, settings_of
+from .errors import ApiError
+from .jobs import run_out
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+CSV = "text/csv; charset=utf-8"
+PDF_MAX_BYTES = 25 * 1024 * 1024
+FORM_SLACK = 64 * 1024
+
+
+# ---------------------------------------------------------------- security settings
+
+
+class SecPut(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    values: dict[str, Any]
+    version: int = Field(ge=0)
+
+
+def _sec_out(conn: Any, request: Request) -> dict[str, Any]:
+    s = settings_of(request)
+    stored, version, updated_at, updated_by = secsettings.load_versioned(conn)
+    defaults = secsettings.defaults(s)
+    return {
+        "values": {**defaults, **stored},
+        "defaults": defaults,
+        "bounds": {k: list(v) for k, v in secsettings.BOUNDS.items()},
+        "version": version,
+        "updated_at": updated_at,
+        "updated_by": str(updated_by) if updated_by else None,
+    }
+
+
+@router.get("/security/settings")
+def security_settings(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    return _sec_out(d.conn, request)
+
+
+@router.put("/security/settings", dependencies=WRITE)
+def security_settings_put(
+    body: SecPut,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    try:
+        secsettings.save(conn, settings_of(request), who=p, values=body.values, version=body.version)
+    except secsettings.SecSettingsError as e:
+        raise ApiError(422, "bad_settings", e.errors[0], extra={"errors": e.errors}) from None
+    except secsettings.VersionConflict:
+        raise ApiError(
+            409,
+            "version_conflict",
+            "Someone else changed the security settings. Reload and try again.",
+            extra={"current": jsonable_encoder(_sec_out(conn, request))},
+        ) from None
+    return _sec_out(conn, request)
+
+
+# ---------------------------------------------------------------- alerts and integrity
+
+
+@router.get("/security/alerts")
+def security_alert_list(
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    hours: int = Query(24, ge=1, le=168),
+) -> list[dict[str, Any]]:
+    return security_alerts.alerts(d.conn, hours)
+
+
+@router.get("/security/integrity")
+def security_integrity(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    with d.conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT * FROM job_runs WHERE job = 'integrity' AND finished_at IS NOT NULL ORDER BY id DESC LIMIT 1"
+        ).fetchone()
+        findings = (
+            cur.execute(
+                "SELECT kind, subject, detail, at FROM integrity_findings WHERE run_id = %s ORDER BY id", (r["id"],)
+            ).fetchall()
+            if r
+            else []
+        )
+    run = None
+    if r is not None:
+        full = run_out(r) or {}
+        run = {k: full.get(k) for k in ("id", "started_at", "finished_at", "status", "counts")}
+    return {
+        "run": run,
+        "findings": [
+            {"kind": f["kind"], "subject": f["subject"], "detail": f["detail"] or {}, "at": f["at"]} for f in findings
+        ],
+    }
+
+
+# ---------------------------------------------------------------- access log
+
+ACCESS_LIMIT_MAX = 1000
+CSV_ROWS_MAX = 50_000
+FAIL_ACTIONS = ("auth.signin_failed", "auth.lockout", "auth.signin_burst")
+RX_CURSOR = re.compile(r"^(?P<at>[0-9T:.+\-Z ]{10,40}),(?P<src>[01]),(?P<id>[0-9]{1,19})$")
+
+ACCESS_SQL = """
+SELECT x.at, x.src, x.id, x.source, x.kind, x.ok, coalesce(x.account, u.sam_account) AS account, x.user_id,
+       u.display_name, host(x.ip) AS ip, x.detail
+FROM (
+    SELECT a.at, 0 AS src, a.id, 'attempt' AS source, a.kind, a.ok, a.account, a.user_id, a.ip,
+           jsonb_strip_nulls(jsonb_build_object('reason', a.reason, 'session_id', a.session_id)) AS detail
+    FROM signin_attempts a
+    UNION ALL
+    SELECT e.at, 1, e.id, 'audit', e.action, NOT (e.action = ANY(%(fail)s)), e.detail->>'account', e.user_id, e.ip,
+           coalesce(e.detail, '{{}}'::jsonb)
+    FROM audit_events e WHERE e.action LIKE ANY(%(prefixes)s)
+) x LEFT JOIN users u ON u.id = x.user_id
+WHERE true{where}
+ORDER BY x.at DESC, x.src DESC, x.id DESC
+LIMIT %(limit)s
+"""
+
+
+def _access_where(
+    *,
+    since: datetime | None,
+    until: datetime | None,
+    user_id: uuid.UUID | None,
+    q: str,
+    kind: str,
+    ok: bool | None,
+    ip: str,
+    before: str,
+) -> tuple[str, dict[str, Any]]:
+    terms: list[str] = []
+    params: dict[str, Any] = {"fail": list(FAIL_ACTIONS), "prefixes": list(LOG_PREFIXES)}
+    if since is not None:
+        terms.append("x.at >= %(since)s")
+        params["since"] = since
+    if until is not None:
+        terms.append("x.at < %(until)s")
+        params["until"] = until
+    if user_id is not None:
+        terms.append("x.user_id = %(user_id)s")
+        params["user_id"] = user_id
+    q = q.strip()
+    if q:
+        pat = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+        terms.append(
+            "(x.account ILIKE %(q)s OR u.sam_account ILIKE %(q)s OR u.display_name ILIKE %(q)s"
+            " OR u.emp_no = %(q_exact)s)"
+        )
+        params["q"], params["q_exact"] = pat, q
+    kind = kind.strip()
+    if kind:
+        terms.append("x.kind = %(kind)s")
+        params["kind"] = kind
+    if ok is not None:
+        terms.append("x.ok = %(ok)s")
+        params["ok"] = ok
+    ip = ip.strip()
+    if ip:
+        try:
+            net = ipaddress.ip_network(ip, strict=False)
+        except ValueError:
+            raise ApiError(422, "invalid_request", "ip must be an address or a network like 10.1.2.0/24.") from None
+        terms.append("x.ip <<= %(ip)s::inet")
+        params["ip"] = str(net)
+    before = before.strip()
+    if before:
+        m = RX_CURSOR.match(before)
+        if m is None:
+            raise ApiError(422, "invalid_request", "before is the next_before value of a previous page.")
+        try:
+            at = datetime.fromisoformat(m["at"].replace("Z", "+00:00"))
+        except ValueError:
+            raise ApiError(422, "invalid_request", "before is the next_before value of a previous page.") from None
+        terms.append("(x.at, x.src, x.id) < (%(b_at)s, %(b_src)s, %(b_id)s)")
+        params.update(b_at=at, b_src=int(m["src"]), b_id=int(m["id"]))
+    return "".join(f" AND {t}" for t in terms), params
+
+
+def _access_rows(conn: Any, where: str, params: dict[str, Any], limit: int) -> list[dict[str, Any]]:
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows: list[dict[str, Any]] = cur.execute(ACCESS_SQL.format(where=where), {**params, "limit": limit}).fetchall()
+    return rows
+
+
+def _access_out(r: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": f"{r['source']}:{r['id']}",
+        "at": r["at"],
+        "source": r["source"],
+        "kind": r["kind"],
+        "ok": bool(r["ok"]),
+        "account": r["account"],
+        "user_id": str(r["user_id"]) if r["user_id"] else None,
+        "display_name": r["display_name"],
+        "ip": r["ip"],
+        "detail": r["detail"] or {},
+    }
+
+
+class AccessFilters(BaseModel):
+    since: datetime | None = None
+    until: datetime | None = None
+    user_id: uuid.UUID | None = None
+    q: str = Field("", max_length=200)
+    kind: str = Field("", max_length=60)
+    ok: bool | None = None
+    ip: str = Field("", max_length=60)
+
+
+def access_filters(
+    since: datetime | None = Query(None),  # noqa: B008
+    until: datetime | None = Query(None),  # noqa: B008
+    user_id: uuid.UUID | None = Query(None),  # noqa: B008
+    q: str = Query("", max_length=200),
+    kind: str = Query("", max_length=60),
+    ok: bool | None = Query(None),
+    ip: str = Query("", max_length=60),
+) -> AccessFilters:
+    return AccessFilters(since=since, until=until, user_id=user_id, q=q, kind=kind, ok=ok, ip=ip)
+
+
+@router.get("/access-log")
+def access_log(
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    f: AccessFilters = Depends(access_filters),  # noqa: B008
+    limit: int = Query(100, ge=1, le=ACCESS_LIMIT_MAX),
+    before: str = Query("", max_length=80),
+) -> dict[str, Any]:
+    where, params = _access_where(**f.model_dump(), before=before)
+    rows = _access_rows(d.conn, where, params, limit)
+    nxt = None
+    if len(rows) == limit:
+        last = rows[-1]
+        nxt = f"{last['at'].astimezone(UTC).isoformat()},{last['src']},{last['id']}"
+    return {"rows": [_access_out(r) for r in rows], "next_before": nxt}
+
+
+CSV_HEADER = ("at", "source", "kind", "ok", "account", "display_name", "user_id", "ip", "detail")
+
+
+@router.get("/access-log.csv")
+def access_log_csv(
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    f: AccessFilters = Depends(access_filters),  # noqa: B008
+) -> Response:
+    conn = d.conn
+    where, params = _access_where(**f.model_dump(), before="")
+    rows = _access_rows(conn, where, params, CSV_ROWS_MAX)
+    lines: list[list[Any]] = [list(CSV_HEADER)]
+    for r in rows:
+        o = _access_out(r)
+        lines.append(
+            [
+                o["at"].astimezone(UTC).isoformat(),
+                o["source"],
+                o["kind"],
+                "yes" if o["ok"] else "no",
+                o["account"],
+                o["display_name"],
+                o["user_id"],
+                o["ip"],
+                json.dumps(o["detail"], sort_keys=True, default=str, separators=(",", ":")),
+            ]
+        )
+    text = tracecsv.to_csv(lines) + "\r\n"
+    filters = {k: (str(v) if v is not None else None) for k, v in f.model_dump().items() if v not in (None, "")}
+    with conn.transaction():
+        conn.execute(
+            "INSERT INTO print_log (by, by_name, session_id, kind, what, pages, ip)"
+            " VALUES (%s, %s, %s, 'access-log-csv', %s, NULL, %s)",
+            (p.user_id, p.user.get("display_name"), p.session_id, f"Access log CSV ({len(rows)} rows)", p.ip),
+        )
+        audit.append(
+            conn,
+            action="admin.access_log_exported",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="access_log",
+            subject_id="csv",
+            detail={"rows": len(rows), "filters": filters, "capped": len(rows) >= CSV_ROWS_MAX},
+            ip=p.ip,
+        )
+    day = datetime.now(UTC).strftime("%Y%m%d")
+    return Response(
+        content=text.encode("utf-8"),
+        media_type=CSV,
+        headers={
+            "Content-Disposition": f'attachment; filename="access-log-{day}.csv"',
+            "X-Content-Type-Options": "nosniff",
+            "Cache-Control": "no-store",
+        },
+    )
+
+
+# ---------------------------------------------------------------- who is in now
+
+
+@router.get("/presence")
+def presence(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    s = secsettings.overlay(conn, settings_of(request))
+    widest = max(s.session_idle_minutes, s.session_desk_minutes)
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT s.id, s.user_id, u.display_name, s.kind::text AS kind, dv.name AS device_name, host(s.ip) AS ip,"
+            " s.created_at, s.last_seen_at, s.locked_since, s.roles_snapshot, now() AS db_now,"
+            " now() - s.last_seen_at AS idle"
+            " FROM sessions s JOIN users u ON u.id = s.user_id LEFT JOIN devices dv ON dv.id = s.device_id"
+            " WHERE s.revoked_at IS NULL AND s.expires_at > now()"
+            " AND s.last_seen_at > now() - make_interval(mins => %s::int)"
+            " ORDER BY lower(u.display_name), s.created_at",
+            (widest,),
+        ).fetchall()
+    streams: dict[str, dict[str, Any]] = {}
+    hub = getattr(request.app.state, "events", None)
+    if hub is not None:
+        for screen, users in hub.presence_by_screen().items():
+            for u in users:
+                streams.setdefault(str(u["user_id"]), {"screen": None if screen == "none" else screen, **u})
+    at = rows[0]["db_now"] if rows else datetime.now(UTC)
+    out = []
+    for r in rows:
+        roles = list((r["roles_snapshot"] or {}).get("roles", []))
+        idle_min, _ = idle_minutes(s, roles)
+        if r["idle"].total_seconds() > idle_min * 60:
+            continue
+        stream = streams.get(str(r["user_id"]))
+        out.append(
+            {
+                "session_id": str(r["id"]),
+                "user_id": str(r["user_id"]),
+                "display_name": r["display_name"],
+                "kind": r["kind"],
+                "device_name": r["device_name"],
+                "ip": r["ip"],
+                "created_at": r["created_at"],
+                "last_seen_at": r["last_seen_at"],
+                "locked": r["locked_since"] is not None,
+                "screen": stream["screen"] if stream else None,
+                "packet_id": stream.get("packet_id") if stream else None,
+            }
+        )
+    return {"at": at, "sessions": out}
+
+
+# ---------------------------------------------------------------- stamp check
+
+RX_SIG_IN_TEXT = re.compile(r"stamp\s+([0-9a-fA-F]{16})\b")
+RX_SIG_ALONE = re.compile(r"^\s*([0-9a-fA-F]{16})\s*$")
+RX_UTC = re.compile(r"(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}) UTC")
+TEXT_MAX = 4000
+PDF_STAMPS_MAX = 50
+
+
+class StampText(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    text: str = Field(max_length=TEXT_MAX)
+
+
+def _squash(text: str) -> str:
+    return "".join(text.split())
+
+
+def _sign_candidates(conn: Any, r: dict[str, Any]) -> list[tuple[Any, str, str | None]]:
+    """(packet_id, signed kind, seal hash) the print could have been signed with, from its print_log kind."""
+    kind, pid = str(r["kind"]), r["packet_id"]
+    itp = f"itp-{r['itp_id']}" if r.get("itp_id") is not None else None
+    if kind == "report-pdf":
+        return [(None, "report", None)]
+    if kind == "itp-pdf":
+        return [(itp, "itp-pdf", None)]
+    if kind == "itp":
+        return [(itp, "print:itp", None)]
+    if kind == "databook-export":
+        seals: list[str | None] = [
+            bytes(x[0]).hex()
+            for x in conn.execute(
+                "SELECT hash FROM packet_seals WHERE packet_id = %s ORDER BY id DESC", (pid,)
+            ).fetchall()
+        ]
+        return [(pid, "databook", s) for s in [*seals, None]]
+    if kind == "traveler":
+        return [(pid, "traveler", None), (pid, "print:traveler", None)]
+    return [(pid, f"print:{kind}", None)]
+
+
+def _minutes(r: dict[str, Any]) -> list[str]:
+    out: list[str] = []
+    m = RX_UTC.search(str(r.get("stamp") or ""))
+    if m:
+        out.append(f"{m[1]}T{m[2]}Z")
+    at = r["at"].astimezone(UTC)
+    out.append(stamps.minute_of(at))
+    return list(dict.fromkeys(out))
+
+
+def _print_out(conn: Any, r: dict[str, Any]) -> dict[str, Any]:
+    label = None
+    if r["packet_id"] is not None:
+        row = conn.execute(
+            "SELECT pr.number, u.lot, p.drawing FROM packets p JOIN units u ON u.id = p.unit_id"
+            " JOIN projects pr ON pr.id = u.project_id WHERE p.id = %s",
+            (r["packet_id"],),
+        ).fetchone()
+        if row:
+            label = " ".join(x for x in (row[0], f"Lot {row[1]}" if row[1] else "", row[2] or "") if x)
+    return {
+        "id": r["id"],
+        "at": r["at"],
+        "by_name": r["by_name"],
+        "kind": r["kind"],
+        "what": r["what"],
+        "pages": r["pages"],
+        "packet_id": r["packet_id"],
+        "packet_label": label,
+    }
+
+
+def check_stamp(conn: Any, key: bytes | None, sig: str | None, shown: str | None) -> dict[str, Any]:
+    """One stamp: the signature, and the text shown up to and including it (None when only the hex was given)."""
+    if not sig:
+        return {"found": False, "valid": False, "reason": "No stamp signature in the text.", "print": None}
+    sig = sig.lower()
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute("SELECT * FROM print_log WHERE stamp_sig = %s ORDER BY id", (sig,)).fetchall()
+    if not rows:
+        return {
+            "found": False,
+            "valid": False,
+            "reason": f"No print has stamp {sig}. The copy is not from Packet Station, or the stamp was changed.",
+            "print": None,
+        }
+    r = rows[0]
+    out = {"found": True, "valid": False, "reason": "", "print": _print_out(conn, r)}
+    if key is None:
+        out["reason"] = "The seal key is not available, so the signature cannot be checked."
+        return out
+    signed = any(
+        stamps.verify_stamp(key, pid, kind, seal, r["by"], minute, sig)
+        for pid, kind, seal in _sign_candidates(conn, r)
+        for minute in _minutes(r)
+    )
+    if not signed:
+        out["reason"] = "The signature does not match the print log. Treat the copy as not controlled."
+        return out
+    if shown is not None and r.get("stamp"):
+        logged, seen = _squash(str(r["stamp"])), _squash(shown)
+        if not (logged.endswith(seen) or seen.endswith(logged)):
+            out["reason"] = "The stamp text differs from the print log. The copy was changed."
+            return out
+    out["valid"] = True
+    out["reason"] = "The stamp matches the print log."
+    return out
+
+
+def _seal_key(request: Request) -> bytes | None:
+    try:
+        return settings_of(request).seal_key()
+    except Exception:
+        return None
+
+
+def _audit_verify(conn: Any, p: Principal, results: list[dict[str, Any]], source: str) -> None:
+    audit.append(
+        conn,
+        action="admin.stamp_verified",
+        user_id=p.user_id,
+        session_id=p.session_id,
+        role_snapshot=p.role_snapshot,
+        subject_type="print",
+        subject_id=",".join(str((x["print"] or {}).get("id", "")) for x in results)[:300] or None,
+        detail={
+            "source": source,
+            "results": [
+                {"found": x["found"], "valid": x["valid"], "print_id": (x["print"] or {}).get("id")} for x in results
+            ],
+        },
+        ip=p.ip,
+    )
+
+
+@router.post("/stamp-verify", dependencies=WRITE)
+def stamp_verify(
+    body: StampText,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    text = body.text
+    sig: str | None = None
+    shown: str | None = None
+    m = RX_SIG_IN_TEXT.search(text)
+    if m is not None:
+        sig, shown = m[1], text[: m.end()]
+    else:
+        alone = RX_SIG_ALONE.match(text)
+        if alone:
+            sig = alone[1]
+    res = check_stamp(d.conn, _seal_key(request), sig, shown)
+    _audit_verify(d.conn, p, [res], "text")
+    return res
+
+
+def pdf_stamps(data: bytes) -> list[tuple[str, str]]:
+    """(signature, the page text up to and including it) for every stamp in a PDF's text, first page first."""
+    from pypdf import PdfReader
+    from pypdf.errors import PdfReadError
+
+    try:
+        reader = PdfReader(io.BytesIO(data))
+        found: dict[str, str] = {}
+        for page in reader.pages:
+            text = page.extract_text() or ""
+            for m in RX_SIG_IN_TEXT.finditer(text):
+                sig = m[1].lower()
+                if sig not in found:
+                    found[sig] = text[: m.end()]
+                if len(found) >= PDF_STAMPS_MAX:
+                    break
+    except (PdfReadError, ValueError, KeyError, TypeError) as e:
+        raise ApiError(422, "bad_pdf", f"The PDF cannot be read: {type(e).__name__}.") from None
+    return list(found.items())
+
+
+@router.post("/stamp-verify/pdf")
+async def stamp_verify_pdf(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    length = request.headers.get("content-length")
+    if length is not None and length.isdigit() and int(length) > PDF_MAX_BYTES + FORM_SLACK:
+        raise ApiError(413, "too_large", "The PDF is larger than 25 MB.")
+    form = await request.form(max_files=1, max_fields=2)
+    try:
+        file = form.get("file")
+        if not isinstance(file, UploadFile):
+            raise ApiError(422, "invalid_request", "Send the PDF in the 'file' field.")
+        data = await file.read(PDF_MAX_BYTES + 1)
+    finally:
+        await form.close()
+    if len(data) > PDF_MAX_BYTES:
+        raise ApiError(413, "too_large", "The PDF is larger than 25 MB.")
+    if not data.startswith(b"%PDF-"):
+        raise ApiError(422, "not_pdf", "The file is not a PDF.")
+    found = await run_in_threadpool(pdf_stamps, data)
+    key = _seal_key(request)
+
+    def check_all() -> list[dict[str, Any]]:
+        results = [check_stamp(d.conn, key, sig, shown) for sig, shown in found]
+        _audit_verify(d.conn, p, results, "pdf")
+        return results
+
+    return {"stamps": await run_in_threadpool(check_all)}
+
+
+# ---------------------------------------------------------------- directory flags
+
+
+@router.get("/directory-flags")
+def directory_flags(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    conn = d.conn
+    with conn.cursor(row_factory=dict_row) as cur:
+        run = cur.execute(
+            "SELECT finished_at FROM job_runs WHERE job = 'ad-reconcile' AND finished_at IS NOT NULL"
+            " ORDER BY id DESC LIMIT 1"
+        ).fetchone()
+        rows = cur.execute(
+            "SELECT id, sam_account, display_name, emp_no, directory_flags FROM users WHERE active"
+            " AND (coalesce((directory_flags->>'emp_missing')::bool, false)"
+            " OR coalesce((directory_flags->>'emp_duplicate')::bool, false))"
+            " ORDER BY emp_no NULLS FIRST, lower(display_name)"
+        ).fetchall()
+        synced = cur.execute("SELECT max(ad_synced_at) AS at FROM users").fetchone()
+    missing: list[dict[str, Any]] = []
+    dupes: dict[str, list[dict[str, Any]]] = {}
+    for r in rows:
+        who = {"user_id": str(r["id"]), "sam_account": r["sam_account"], "display_name": r["display_name"]}
+        flags = r["directory_flags"] or {}
+        if flags.get("emp_missing"):
+            missing.append(who)
+        if flags.get("emp_duplicate") and r["emp_no"]:
+            dupes.setdefault(str(r["emp_no"]), []).append(who)
+    return {
+        "checked_at": (run or {}).get("finished_at") or (synced or {}).get("at"),
+        "missing": missing,
+        "duplicate": [{"emp_no": k, "users": v} for k, v in sorted(dupes.items())],
+    }

server/packetstation/api/deps.py

@@ -15,7 +15,7 @@ from fastapi import Depends, Request, Response
 from psycopg.types.json import Jsonb
 
 from .. import audit, db, events
-from ..auth import devices, sessions
+from ..auth import devices, secsettings, sessions
 from ..auth import directory as userdir
 from ..auth import groups as groupmap
 from ..auth.ldap import Directory, DirectoryProtocol, LdapUnavailable
@@ -48,6 +48,15 @@ def settings_of(request: Request) -> Settings:
     return s
 
 
+def effective_settings(request: Request, conn: Any) -> Settings:
+    """The settings with the Admin's security values applied, read once per request."""
+    s: Settings | None = getattr(request.state, "sec_settings", None)
+    if s is None:
+        s = secsettings.overlay(conn, settings_of(request))
+        request.state.sec_settings = s
+    return s
+
+
 def directory_of(request: Request) -> DirectoryProtocol:
     d: DirectoryProtocol | None = request.app.state.directory
     if d is None:
@@ -257,9 +266,9 @@ def resolve(request: Request, d: Db, *, allow_locked: bool, token: str | None =
     token = token or request.cookies.get(sessions.SESSION_COOKIE)
     if not token:
         raise ApiError(401, "no_session", "Not signed in.")
-    s = settings_of(request)
     key = session_key(request)
     conn = d.conn
+    s = effective_settings(request, conn)
     ip = ip_of(request)
     row = sessions.load(conn, key, token)
     if row is None or row["revoked_at"] is not None or row["expires_at"] <= row["db_now"]:
@@ -312,7 +321,7 @@ def resolve(request: Request, d: Db, *, allow_locked: bool, token: str | None =
     if now - row["roles_checked_at"] > timedelta(minutes=s.roles_refresh_minutes):
         roles, groups = _refresh_roles(conn, directory_of(request), row, ip)
     device = devices.get(conn, row["device_id"]) if row.get("device_id") else None
-    return Principal(
+    p = Principal(
         session=row,
         user=user,
         roles=roles,
@@ -324,6 +333,9 @@ def resolve(request: Request, d: Db, *, allow_locked: bool, token: str | None =
         ip=ip,
         lock_minutes=idle_minutes(s, roles)[1],
     )
+    # The error handler audits a 403 against the principal it was refused to (policy.denied)
+    request.state.principal = p
+    return p
 
 
 def current_principal(request: Request, d: Db = Depends(get_db)) -> Principal:  # noqa: B008

server/packetstation/api/errors.py

@@ -49,7 +49,22 @@ def error_response(err: ApiError) -> JSONResponse:
 
 def install(app: FastAPI) -> None:
     @app.exception_handler(ApiError)
-    async def _api_error(_: Request, exc: ApiError) -> JSONResponse:
+    async def _api_error(request: Request, exc: ApiError) -> JSONResponse:
+        principal = getattr(request.state, "principal", None)
+        if exc.status == 403 and principal is not None:
+            # A refusal to a signed-in person is audited as policy.denied: method, path and code, never the body
+            from starlette.concurrency import run_in_threadpool
+
+            from ..domain.security_alerts import record_denied
+
+            await run_in_threadpool(
+                record_denied,
+                request.app.state.settings,
+                principal,
+                method=request.method,
+                path=request.url.path,
+                code=exc.code,
+            )
         return error_response(exc)
 
     @app.exception_handler(RequestValidationError)

server/packetstation/api/itp.py

@@ -493,6 +493,7 @@ def reject_line(
         )
         head = f"NCR {ncr}" if ncr else "NCR # pending"
         w.log(f"ITP line {key} rejected", f"{head} · {note}" if note else head)
+        dom.notify_reject(w, key, ncr, note)
     return _answer(d, p, itp_id)
 
 

server/packetstation/api/notifications.py

@@ -0,0 +1,511 @@
+"""Notifications: the Admin's routing rules, test send and sent log, and each person's own view on Account.
+
+Who gets what lives in the database (notification_rules, _recipients, _exclusions), never in the environment.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from datetime import time
+from typing import Any, Literal, cast
+
+from fastapi import APIRouter, Depends, Query, Request
+from psycopg.rows import dict_row
+from pydantic import BaseModel, ConfigDict, Field
+
+from .. import audit
+from ..auth.ldap import LdapUnavailable
+from ..notify import seed as seed_rules
+from ..notify import send as notify_send
+from ..notify import smtp
+from ..notify.catalog import EVENT_TYPES, SECURITY_TYPES
+from ..notify.routing import (
+    DIRECTORY_DOWN,
+    NotifyDirectory,
+    Rule,
+    address_ok,
+    cn_of,
+    dn_under,
+    load_rules,
+    supervisor_depts,
+)
+from ..settings import Settings
+from ..text import clean_text
+from .deps import Db, Principal, admin_principal, current_principal, directory_of, get_db, password_principal
+from .errors import ApiError
+from .packets import WRITE
+
+log = logging.getLogger(__name__)
+
+router = APIRouter(tags=["notifications"])
+
+ADMIN = "/api/admin/notifications"
+ACCOUNT = "/api/account/notifications"
+RECIPIENTS_MAX = 50
+EXCLUSIONS_MAX = 200
+TIME_RE = re.compile(r"^([01][0-9]|2[0-3]):([0-5][0-9])$")
+NAME_MAX = 256
+DN_MAX = 1024
+
+
+def _directory(request: Request) -> NotifyDirectory:
+    return cast(NotifyDirectory, directory_of(request))
+
+
+def _settings(request: Request) -> Settings:
+    s: Settings = request.app.state.settings
+    return s
+
+
+def _known(event_type: str) -> None:
+    if event_type not in EVENT_TYPES:
+        raise ApiError(404, "not_found", "No such notification.")
+
+
+# ---------------------------------------------------------------- views
+
+
+def rule_view(conn: Any, rule: Rule) -> dict[str, Any]:
+    et = EVENT_TYPES[rule.event_type]
+    last = conn.execute(
+        "SELECT at, result FROM notifications_sent WHERE event_type = %s ORDER BY id DESC LIMIT 1",
+        (rule.event_type,),
+    ).fetchone()
+    return {
+        "event_type": rule.event_type,
+        "label": et.label,
+        "description": et.description,
+        "security": et.security,
+        "dept_scoped": et.dept_scoped,
+        "param": {"name": et.param, "value": rule.param_value, "default": et.param_default} if et.param else None,
+        "enabled": rule.enabled,
+        "delivery": rule.delivery,
+        "digest_time": rule.digest_time.strftime("%H:%M"),
+        "recipients": [
+            {
+                "id": r["id"],
+                "kind": r["kind"],
+                "group_dn": r["group_dn"],
+                "group_name": r["group_name"],
+                "user_id": str(r["user_id"]) if r["user_id"] else None,
+                "display_name": r.get("display_name"),
+                "address": r["address"],
+            }
+            for r in rule.recipients
+        ],
+        "exclusions": [
+            {"user_id": str(x["user_id"]), "display_name": x.get("display_name"), "self": bool(x["self"])}
+            for x in rule.exclusions
+        ],
+        "version": rule.version,
+        "last_sent": {"at": last[0], "result": last[1]} if last else None,
+    }
+
+
+def _rules(conn: Any, settings: Settings) -> dict[str, Rule]:
+    seed_rules(conn, settings)
+    return load_rules(conn)
+
+
+@router.get(ADMIN)
+def admin_rules(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    s = _settings(request)
+    rules = _rules(d.conn, s)
+    return {
+        "dry_run": bool(s.notify_dry_run),
+        "smtp": {"host": s.smtp_host, "port": s.smtp_port, "from": s.smtp_from, "starttls": s.smtp_starttls},
+        "subject_prefix": smtp.subject_prefix(s),
+        "rules": [rule_view(d.conn, rules[k]) for k in EVENT_TYPES if k in rules],
+    }
+
+
+# ---------------------------------------------------------------- update a rule
+
+
+class RecipientIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    kind: Literal["group", "user", "mailbox", "dept_supervisor"]
+    group_dn: str | None = Field(default=None, max_length=DN_MAX)
+    group_name: str | None = Field(default=None, max_length=NAME_MAX)
+    user_id: uuid.UUID | None = None
+    address: str | None = Field(default=None, max_length=254)
+
+
+class RuleIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool
+    delivery: Literal["immediate", "digest"]
+    digest_time: str = Field(max_length=5)
+    param: int | None = Field(default=None, ge=1, le=10000)
+    recipients: list[RecipientIn] = Field(max_length=RECIPIENTS_MAX)
+    exclusions: list[uuid.UUID] = Field(default_factory=list, max_length=EXCLUSIONS_MAX)
+    version: int
+
+
+def _invalid(text: str) -> ApiError:
+    return ApiError(422, "invalid_request", text)
+
+
+def _clean_recipients(conn: Any, s: Settings, event_type: str, items: list[RecipientIn]) -> list[dict[str, Any]]:
+    et = EVENT_TYPES[event_type]
+    base = s.ldap_base_dn.strip() or s.ldap_group_base_dn.strip()
+    out: list[dict[str, Any]] = []
+    seen: set[tuple[str, str]] = set()
+    for r in items:
+        row: dict[str, Any] = {"kind": r.kind, "group_dn": None, "group_name": None, "user_id": None, "address": None}
+        if r.kind == "group":
+            dn = clean_text(r.group_dn or "", DN_MAX)
+            if not dn or not dn.lower().startswith("cn="):
+                raise _invalid("A group needs its DN.")
+            if base and not dn_under(dn, base):
+                raise _invalid("The group must be in the domain (under LDAP_BASE_DN).")
+            row["group_dn"] = dn
+            row["group_name"] = clean_text(r.group_name or "", NAME_MAX) or cn_of(dn)
+            key = ("group", dn.lower())
+        elif r.kind == "user":
+            if r.user_id is None:
+                raise _invalid("A user recipient needs a user.")
+            if conn.execute("SELECT 1 FROM users WHERE id = %s", (r.user_id,)).fetchone() is None:
+                raise _invalid("No such user.")
+            row["user_id"] = r.user_id
+            key = ("user", str(r.user_id))
+        elif r.kind == "mailbox":
+            addr = (r.address or "").strip()
+            if not address_ok(addr):
+                raise _invalid(f"Not an email address: {addr[:80]}")
+            row["address"] = addr
+            key = ("mailbox", addr.lower())
+        else:
+            if not et.dept_scoped:
+                raise _invalid("This notification has no department, so it has no Supervisor group.")
+            key = ("dept_supervisor", "")
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(row)
+    return out
+
+
+@router.put(f"{ADMIN}/{{event_type}}", dependencies=WRITE)
+def update_rule(
+    event_type: str,
+    body: RuleIn,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    _known(event_type)
+    s = _settings(request)
+    conn = d.conn
+    et = EVENT_TYPES[event_type]
+    m = TIME_RE.match(body.digest_time)
+    if m is None:
+        raise _invalid("The digest time is HH:MM.")
+    digest_at = time(int(m.group(1)), int(m.group(2)))
+    if body.param is not None and et.param is None:
+        raise _invalid("This notification takes no setting.")
+    excl = list(dict.fromkeys(body.exclusions))
+    if excl and event_type in SECURITY_TYPES:
+        raise _invalid("Nobody can be left out of a security notification.")
+    recipients = _clean_recipients(conn, s, event_type, body.recipients)
+    if excl:
+        found = {r[0] for r in conn.execute("SELECT id FROM users WHERE id = ANY(%s)", (excl,)).fetchall()}
+        if len(found) != len(excl):
+            raise _invalid("An excluded person is not a known user.")
+    _rules(conn, s)
+    with conn.transaction():
+        before_rule = load_rules(conn, event_type).get(event_type)
+        assert before_rule is not None
+        before = rule_view(conn, before_rule)
+        done = conn.execute(
+            "UPDATE notification_rules SET enabled = %s, delivery = %s, digest_time = %s, param = %s,"
+            " version = version + 1, updated_at = now(), updated_by = %s WHERE event_type = %s AND version = %s"
+            " RETURNING version",
+            (
+                body.enabled,
+                body.delivery,
+                digest_at,
+                (body.param or et.param_default) if et.param else None,
+                p.user_id,
+                event_type,
+                body.version,
+            ),
+        ).fetchone()
+        if done is None:
+            raise ApiError(
+                409,
+                "version_conflict",
+                "Someone else changed this notification. Here is the current one.",
+                extra={"current": before},
+            )
+        conn.execute("DELETE FROM notification_recipients WHERE event_type = %s", (event_type,))
+        for r in recipients:
+            conn.execute(
+                "INSERT INTO notification_recipients (event_type, kind, group_dn, group_name, user_id, address)"
+                " VALUES (%s, %s, %s, %s, %s, %s)",
+                (event_type, r["kind"], r["group_dn"], r["group_name"], r["user_id"], r["address"]),
+            )
+        conn.execute(
+            "DELETE FROM notification_exclusions WHERE event_type = %s AND NOT self AND NOT (user_id = ANY(%s))",
+            (event_type, excl),
+        )
+        for uid in excl:
+            conn.execute(
+                "INSERT INTO notification_exclusions (event_type, user_id, self, created_by) VALUES (%s, %s, false, %s)"
+                " ON CONFLICT (event_type, user_id) DO NOTHING",
+                (event_type, uid, p.user_id),
+            )
+        after_rule = load_rules(conn, event_type)[event_type]
+        after = rule_view(conn, after_rule)
+        audit.append(
+            conn,
+            action="admin.notification_rule_updated",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="notification_rule",
+            subject_id=event_type,
+            detail={"event_type": event_type, "before": _audit_view(before), "after": _audit_view(after)},
+            ip=p.ip,
+        )
+    return after
+
+
+def _audit_view(v: dict[str, Any]) -> dict[str, Any]:
+    keep = ("enabled", "delivery", "digest_time", "param", "version")
+    out = {k: v[k] for k in keep}
+    out["recipients"] = [
+        {k: r[k] for k in ("kind", "group_dn", "group_name", "user_id", "address") if r.get(k)} for r in v["recipients"]
+    ]
+    out["exclusions"] = [{"user_id": x["user_id"], "self": x["self"]} for x in v["exclusions"]]
+    return out
+
+
+# ---------------------------------------------------------------- group search, test send, sent log
+
+
+@router.get(f"{ADMIN}/groups")
+def search_groups(
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    q: str = Query(..., min_length=2, max_length=100),
+) -> list[dict[str, Any]]:
+    try:
+        found = _directory(request).search_groups(q.strip(), 25)
+    except LdapUnavailable as e:
+        log.warning("group search: AD unavailable: %s", e)
+        raise ApiError(503, "ldap_unavailable", "The directory is not available. Try again in a minute.") from None
+    return [{"dn": g.get("dn"), "name": g.get("name"), "description": g.get("description")} for g in found]
+
+
+class TestIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    to: str | None = Field(default=None, max_length=254)
+
+
+@router.post(f"{ADMIN}/{{event_type}}/test", dependencies=WRITE)
+def test_send(
+    event_type: str,
+    body: TestIn,
+    request: Request,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    _known(event_type)
+    s = _settings(request)
+    to = (body.to or "").strip() or None
+    if to is not None and not address_ok(to):
+        raise _invalid("Not an email address.")
+    conn = d.conn
+    _rules(conn, s)
+    out = notify_send.send_test(conn, s, _directory(request), event_type, to=to, requested_by=p.user_id)
+    audit.append(
+        conn,
+        action="admin.notification_test",
+        user_id=p.user_id,
+        session_id=p.session_id,
+        role_snapshot=p.role_snapshot,
+        subject_type="notification_rule",
+        subject_id=event_type,
+        detail={
+            "event_type": event_type,
+            "to": to,
+            "result": out["result"],
+            "recipients": len(out["recipients"]),
+            "sent_id": out["sent_id"],
+        },
+        ip=p.ip,
+    )
+    return out
+
+
+@router.get(f"{ADMIN}/sent")
+def sent_log(
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    event_type: str | None = Query(None, max_length=64),
+    limit: int = Query(50, ge=1, le=500),
+    before: int | None = Query(None, ge=1),
+) -> dict[str, Any]:
+    sql = (
+        "SELECT id, at, event_type, digest, test, subject, recipients, result, error FROM notifications_sent"
+        " WHERE (%s::text IS NULL OR event_type = %s) AND (%s::bigint IS NULL OR id < %s) ORDER BY id DESC LIMIT %s"
+    )
+    args = (event_type or None, event_type or None, before, before)
+    with d.conn.cursor(row_factory=dict_row) as cur:
+        rows = [dict(r) for r in cur.execute(sql, (*args, limit)).fetchall()]
+    return {"rows": rows, "next_before": rows[-1]["id"] if len(rows) == limit else None}
+
+
+# ---------------------------------------------------------------- account
+
+
+def _my_groups(directory: NotifyDirectory, user_id: uuid.UUID) -> tuple[Any, set[str]]:
+    """(the AdUser or None, lower-case names of the user's groups in the PacketStation OU, nested)."""
+    u = directory.get_user_by_guid(user_id)
+    if u is None:
+        return None, set()
+    return u, {g.lower() for g in directory.groups_of(u.dn)}
+
+
+def account_view(conn: Any, settings: Settings, directory: NotifyDirectory, p: Principal) -> list[dict[str, Any]]:
+    rules = _rules(conn, settings)
+    email = str(p.user.get("email") or "").strip().lower()
+    me = str(p.user_id)
+    sup = supervisor_depts(p.roles)
+    groups: tuple[Any, set[str]] | None = None
+    down = False
+    member_cache: dict[str, bool] = {}
+    out: list[dict[str, Any]] = []
+    for key, et in EVENT_TYPES.items():
+        rule = rules.get(key)
+        if rule is None or not rule.enabled:
+            continue
+        why: list[str] = []
+        for r in rule.recipients:
+            kind = r["kind"]
+            if kind == "user" and str(r["user_id"]) == me:
+                why.append("user")
+            elif kind == "mailbox" and email and str(r["address"]).strip().lower() == email:
+                why.append("mailbox")
+            elif kind == "dept_supervisor" and et.dept_scoped and sup:
+                why.extend(f"Supervisor group of {code}" for code in sup)
+            elif kind == "group" and not down:
+                dn = str(r["group_dn"])
+                name = str(r["group_name"] or cn_of(dn))
+                try:
+                    if dn_under(dn, settings.ldap_group_base_dn):
+                        if groups is None:
+                            groups = _my_groups(directory, p.user_id)
+                        hit = name.lower() in groups[1]
+                    else:
+                        if dn.lower() not in member_cache:
+                            member_cache[dn.lower()] = any(
+                                u.guid == p.user_id for u in directory.members_of_dn(dn) if not u.disabled
+                            )
+                        hit = member_cache[dn.lower()]
+                except Exception as e:
+                    log.warning("account notifications: directory: %s", type(e).__name__)
+                    down = True
+                    continue
+                if hit:
+                    why.append(f"group {name}")
+        excl = next((x for x in rule.exclusions if str(x["user_id"]) == me), None)
+        opted_out = bool(excl and excl["self"])
+        if excl and not excl["self"]:
+            why.append("left out by an Admin")
+        if down and any(r["kind"] == "group" for r in rule.recipients):
+            why.append(DIRECTORY_DOWN)
+        reaches = excl is None and any(w not in (DIRECTORY_DOWN, "left out by an Admin") for w in why)
+        out.append(
+            {
+                "event_type": key,
+                "label": et.label,
+                "security": et.security,
+                "delivery": rule.delivery,
+                "reaches": reaches,
+                "why": list(dict.fromkeys(why)),
+                "opted_out": opted_out,
+                "can_opt_out": not et.security,
+            }
+        )
+    return out
+
+
+@router.get(ACCOUNT)
+def my_notifications(
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> list[dict[str, Any]]:
+    return account_view(d.conn, _settings(request), _directory(request), p)
+
+
+class OptOutIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    opt_out: bool
+
+
+@router.put(f"{ACCOUNT}/{{event_type}}", dependencies=WRITE)
+def opt_out(
+    event_type: str,
+    body: OptOutIn,
+    request: Request,
+    p: Principal = Depends(password_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    _known(event_type)
+    if event_type in SECURITY_TYPES:
+        raise ApiError(403, "forbidden", "Security notifications cannot be turned off.")
+    conn = d.conn
+    s = _settings(request)
+    _rules(conn, s)
+    with conn.transaction():
+        if body.opt_out:
+            changed = conn.execute(
+                "INSERT INTO notification_exclusions (event_type, user_id, self, created_by) VALUES (%s, %s, true, %s)"
+                " ON CONFLICT (event_type, user_id) DO NOTHING",
+                (event_type, p.user_id, p.user_id),
+            ).rowcount
+        else:
+            changed = conn.execute(
+                "DELETE FROM notification_exclusions WHERE event_type = %s AND user_id = %s AND self",
+                (event_type, p.user_id),
+            ).rowcount
+        if changed:
+            audit.append(
+                conn,
+                action="account.notification_opt_out",
+                user_id=p.user_id,
+                session_id=p.session_id,
+                role_snapshot=p.role_snapshot,
+                subject_type="notification_rule",
+                subject_id=event_type,
+                detail={"event_type": event_type, "opt_out": body.opt_out},
+                ip=p.ip,
+            )
+    item = next((x for x in account_view(conn, s, _directory(request), p) if x["event_type"] == event_type), None)
+    if item is None:
+        # The rule is off: nothing reaches anyone, but the choice is kept
+        et = EVENT_TYPES[event_type]
+        item = {
+            "event_type": event_type,
+            "label": et.label,
+            "security": False,
+            "delivery": None,
+            "reaches": False,
+            "why": [],
+            "opted_out": body.opt_out,
+            "can_opt_out": True,
+        }
+    return item

server/packetstation/auth/directory.py

@@ -83,13 +83,43 @@ def get_user(conn: Any, user_id: Any) -> dict[str, Any] | None:
     return row
 
 
-def mark_inactive(conn: Any, user_id: Any, *, reason: str, gone: bool) -> None:
+def end_sessions(conn: Any, user_id: Any, reason: str, *, by: str | None = None) -> list[uuid.UUID]:
+    """End every live session of a user, one `auth.session_ended` audit row each. Returns the session ids."""
+    rows = conn.execute(
+        "SELECT id, roles_snapshot FROM sessions WHERE user_id = %s AND revoked_at IS NULL", (user_id,)
+    ).fetchall()
+    ended: list[uuid.UUID] = []
+    for sid, snap in rows:
+        if not sessions.revoke(conn, sid, reason):
+            continue
+        ended.append(sid)
+        detail: dict[str, Any] = {"reason": reason}
+        if by:
+            detail["by"] = by
+        audit.append(
+            conn,
+            action="auth.session_ended",
+            user_id=user_id,
+            session_id=sid,
+            role_snapshot=",".join((snap or {}).get("roles", [])),
+            detail=detail,
+        )
+    return ended
+
+
+def mark_inactive(conn: Any, user_id: Any, *, reason: str, gone: bool, audit_by: str | None = None) -> list[uuid.UUID]:
+    """Mark a user inactive (disabled or gone in AD), drop the roles and end the sessions.
+
+    With `audit_by` every ended session gets an `auth.session_ended` row naming who ended it. Returns the ids.
+    """
     conn.execute(
         "UPDATE users SET active = false, directory_flags = directory_flags || %s, ad_synced_at = now() WHERE id = %s",
         (Jsonb({"disabled": not gone, "gone": gone}), user_id),
     )
     conn.execute("DELETE FROM user_roles WHERE user_id = %s", (user_id,))
-    sessions.revoke_user(conn, user_id, reason)
+    if audit_by:
+        return end_sessions(conn, user_id, reason, by=audit_by)
+    return sessions.revoke_user(conn, user_id, reason)
 
 
 def public_user(row: dict[str, Any]) -> dict[str, Any]:
@@ -115,133 +145,195 @@ def ad_user_public(u: AdUser) -> dict[str, Any]:
     }
 
 
-# ---------------------------------------------------------------- AD refresh
+# ---------------------------------------------------------------- AD reconcile
+
+RECONCILE_BY = "ad-reconcile"
+LIST_CAP = 200  # names kept in a report list; the count is always whole
+
+
+class ReconcileRefused(Exception):
+    """AD answered, but with something that would deactivate people by mistake. Nothing was changed."""
 
 
 @dataclass
 class RefreshResult:
     groups_checked: int = 0
     groups_missing: list[str] = field(default_factory=list)
+    groups_extra: list[str] = field(default_factory=list)
     users_seen: int = 0
     created: int = 0
     updated: int = 0
     deactivated: int = 0
+    sessions_ended: int = 0
     emp_missing: list[str] = field(default_factory=list)
     emp_duplicate: list[str] = field(default_factory=list)
+    created_names: list[str] = field(default_factory=list)
+    deactivated_names: list[str] = field(default_factory=list)
 
     def as_dict(self) -> dict[str, Any]:
+        """The report: lists of names (missing and extra groups, accounts without an employee ID, duplicated
+        employee IDs) and counts. The same shape is the ad-reconcile run's counts."""
         return {
             "groups_checked": self.groups_checked,
-            "groups_missing": self.groups_missing,
+            "groups_missing": self.groups_missing[:LIST_CAP],
+            "groups_extra": self.groups_extra[:LIST_CAP],
             "users_seen": self.users_seen,
             "created": self.created,
             "updated": self.updated,
             "deactivated": self.deactivated,
-            "emp_missing": self.emp_missing,
-            "emp_duplicate": self.emp_duplicate,
+            "sessions_ended": self.sessions_ended,
+            "emp_missing": self.emp_missing[:LIST_CAP],
+            "emp_duplicate": self.emp_duplicate[:LIST_CAP],
+        }
+
+    def summary(self) -> dict[str, int]:
+        """Counts only, for the audit row and the log."""
+        return {
+            "groups_checked": self.groups_checked,
+            "groups_missing": len(self.groups_missing),
+            "groups_extra": len(self.groups_extra),
+            "users_seen": self.users_seen,
+            "created": self.created,
+            "updated": self.updated,
+            "deactivated": self.deactivated,
+            "sessions_ended": self.sessions_ended,
+            "emp_missing": len(self.emp_missing),
+            "emp_duplicate": len(self.emp_duplicate),
         }
 
 
-def refresh_from_ad(
-    conn: Any, directory: DirectoryProtocol, *, actor: sessions.Actor | None = None, ip: str | None = None
-) -> RefreshResult:
-    """Enumerate every mapped group, upsert its members and rebuild their roles; deactivate gone accounts.
+@dataclass
+class _AdRead:
+    """Everything the reconcile reads from AD, before any database write."""
 
-    All AD reads happen first, then the database writes in one transaction, so no lock is held while AD
-    answers. Raises LdapUnavailable when AD cannot be read.
-    """
-    res = RefreshResult()
+    present: list[tuple[str, str]]  # mapped groups found in the OU: (name as mapped, dn)
+    missing: list[str]
+    extra: list[str]
+    seen: dict[uuid.UUID, AdUser]
+    member_groups: dict[uuid.UUID, set[str]]
+    unseen: dict[uuid.UUID, AdUser | None]
+
+
+def _read_ad(conn: Any, directory: DirectoryProtocol) -> _AdRead:
     names = groupmap.group_names()
-    in_ad = {g.lower() for g in directory.list_groups()}
-    res.groups_missing = [n for n in names if n.lower() not in in_ad]
-    present = [n for n in names if n.lower() in in_ad]
+    in_ou = {cn.lower(): (cn, dn) for cn, dn in directory.list_groups_detail()}
+    mapped = {n.lower() for n in names}
+    present = [(n, in_ou[n.lower()][1]) for n in names if n.lower() in in_ou]
+    missing = [n for n in names if n.lower() not in in_ou]
+    extra = sorted((cn for key, (cn, _) in in_ou.items() if key not in mapped), key=str.lower)
+    if not present:
+        raise ReconcileRefused("no mapped group was found in the OU; check LDAP_GROUP_BASE_DN")
     seen: dict[uuid.UUID, AdUser] = {}
     member_groups: dict[uuid.UUID, set[str]] = {}
-    for name in present:
-        for u in directory.members_of(name):
+    for name, dn in present:
+        for u in directory.members_of_dn(dn):
             seen.setdefault(u.guid, u)
             member_groups.setdefault(u.guid, set()).add(name)
-    res.groups_checked = len(present)
-    res.users_seen = len(seen)
+    known = [r[0] for r in conn.execute("SELECT id FROM users WHERE active ORDER BY id").fetchall()]
+    unseen_ids = [uid for uid in known if uid not in seen]
+    if not seen and unseen_ids:
+        raise ReconcileRefused("AD listed no member in any mapped group; nobody was changed")
+    unseen = directory.users_by_guid(unseen_ids) if unseen_ids else {}
+    return _AdRead(present, missing, extra, seen, member_groups, unseen)
 
-    known = conn.execute("SELECT id FROM users WHERE active").fetchall()
-    unseen = [r[0] for r in known if r[0] not in seen]
-    # Accounts not in any group: still in AD (keep, no roles), disabled, or gone
-    unseen_state: dict[uuid.UUID, AdUser | None] = {uid: directory.get_user_by_guid(uid) for uid in unseen}
 
-    enabled = [u for u in seen.values() if not u.disabled]
+def refresh_from_ad(
+    conn: Any,
+    directory: DirectoryProtocol,
+    *,
+    actor: sessions.Actor | None = None,
+    ip: str | None = None,
+    run_id: int | None = None,
+) -> RefreshResult:
+    """The AD reconcile. List the OU (missing and extra groups), enumerate the members of every mapped group
+    (IN_CHAIN, paged), upsert them on objectGUID and rebuild their roles. An active user AD no longer lists in
+    any group is looked up by GUID: gone or disabled means inactive with the sessions ended (`ad_gone`,
+    `ad_disabled`); still there means no roles and the sessions ended (`roles_lost`). directory_flags gets
+    emp_missing, emp_duplicate, gone and disabled.
+
+    Every AD read happens first, then the writes in one transaction, so no lock is held while AD answers.
+    Raises LdapUnavailable when AD cannot be read and ReconcileRefused when what it read looks wrong.
+    """
+    res = RefreshResult()
+    ad = _read_ad(conn, directory)
+    res.groups_missing = ad.missing
+    res.groups_extra = ad.extra
+    res.groups_checked = len(ad.present)
+    res.users_seen = len(ad.seen)
+
+    enabled = [u for u in ad.seen.values() if not u.disabled]
+    enabled += [u for u in ad.unseen.values() if u is not None and not u.disabled]
     res.emp_missing = sorted(u.sam_account for u in enabled if not u.emp_no)
     counts = Counter(u.emp_no for u in enabled if u.emp_no)
     res.emp_duplicate = sorted(e for e, n in counts.items() if n > 1)
     dup = set(res.emp_duplicate)
+    who: dict[str, Any] = {
+        "user_id": actor.user_id if actor else None,
+        "session_id": actor.session_id if actor else None,
+        "role_snapshot": actor.role_snapshot if actor else None,
+    }
 
-    with conn.transaction():
-        for guid, u in seen.items():
-            grp = groupmap.mapped_groups(member_groups[guid])
-            was_active_row = conn.execute("SELECT active FROM users WHERE id = %s", (guid,)).fetchone()
-            created = upsert_user(conn, u, groups=grp)
-            conn.execute(
-                "UPDATE users SET directory_flags = directory_flags || %s WHERE id = %s",
-                (Jsonb({"emp_duplicate": bool(u.emp_no and u.emp_no in dup), "gone": False}), guid),
-            )
-            if created:
-                res.created += 1
-            else:
-                res.updated += 1
-            if u.disabled:
-                set_roles(conn, guid, [])
-                sessions.revoke_user(conn, guid, "ad_disabled")
-                if was_active_row is not None and was_active_row[0]:
-                    res.deactivated += 1
-                    audit.append(
-                        conn,
-                        action="ad.user_deactivated",
-                        user_id=actor.user_id if actor else None,
-                        session_id=actor.session_id if actor else None,
-                        role_snapshot=actor.role_snapshot if actor else None,
-                        subject_type="user",
-                        subject_id=guid,
-                        detail={"reason": "disabled", "sam_account": u.sam_account},
-                        ip=ip,
-                    )
-            else:
-                set_roles(conn, guid, groupmap.roles_from_groups(grp))
-        for uid, found in unseen_state.items():
-            if found is None or found.disabled:
-                mark_inactive(conn, uid, reason="ad_disabled" if found else "ad_gone", gone=found is None)
-                res.deactivated += 1
-                audit.append(
-                    conn,
-                    action="ad.user_deactivated",
-                    user_id=actor.user_id if actor else None,
-                    session_id=actor.session_id if actor else None,
-                    role_snapshot=actor.role_snapshot if actor else None,
-                    subject_type="user",
-                    subject_id=uid,
-                    detail={"reason": "gone" if found is None else "disabled"},
-                    ip=ip,
-                )
-            else:
-                upsert_user(conn, found, groups=[])
-                set_roles(conn, uid, [])
-                sessions.revoke_user(conn, uid, "roles_lost")
+    def deactivated(uid: uuid.UUID, reason: str, sam: str) -> None:
+        res.deactivated += 1
+        res.deactivated_names.append(sam)
         audit.append(
             conn,
-            action="ad.refresh",
-            user_id=actor.user_id if actor else None,
-            session_id=actor.session_id if actor else None,
-            role_snapshot=actor.role_snapshot if actor else None,
-            detail={
-                "groups_checked": res.groups_checked,
-                "groups_missing": len(res.groups_missing),
-                "users_seen": res.users_seen,
-                "created": res.created,
-                "updated": res.updated,
-                "deactivated": res.deactivated,
-                "emp_missing": len(res.emp_missing),
-                "emp_duplicate": len(res.emp_duplicate),
-            },
+            action="ad.user_deactivated",
+            subject_type="user",
+            subject_id=uid,
+            detail={"reason": reason, "sam_account": sam},
             ip=ip,
+            **who,
         )
-    log.info("AD refresh: %s", {k: v for k, v in res.as_dict().items() if isinstance(v, int)})
+
+    def flags(uid: uuid.UUID, u: AdUser | None) -> None:
+        conn.execute(
+            "UPDATE users SET directory_flags = directory_flags || %s WHERE id = %s",
+            (Jsonb({"emp_duplicate": bool(u and u.emp_no and u.emp_no in dup), "gone": u is None}), uid),
+        )
+
+    with conn.transaction():
+        for guid, u in ad.seen.items():
+            grp = groupmap.mapped_groups(ad.member_groups[guid])
+            was = conn.execute("SELECT active FROM users WHERE id = %s", (guid,)).fetchone()
+            if upsert_user(conn, u, groups=grp):
+                res.created += 1
+                res.created_names.append(u.sam_account)
+            else:
+                res.updated += 1
+            flags(guid, u)
+            roles = [] if u.disabled else groupmap.roles_from_groups(grp)
+            set_roles(conn, guid, roles)
+            if u.disabled:
+                res.sessions_ended += len(end_sessions(conn, guid, "ad_disabled", by=RECONCILE_BY))
+                if was is not None and was[0]:
+                    deactivated(guid, "disabled", u.sam_account)
+            elif not roles:
+                res.sessions_ended += len(end_sessions(conn, guid, "roles_lost", by=RECONCILE_BY))
+        for uid, found in ad.unseen.items():
+            if found is None or found.disabled:
+                row = conn.execute("SELECT sam_account FROM users WHERE id = %s", (uid,)).fetchone()
+                ended = mark_inactive(
+                    conn,
+                    uid,
+                    reason="ad_gone" if found is None else "ad_disabled",
+                    gone=found is None,
+                    audit_by=RECONCILE_BY,
+                )
+                res.sessions_ended += len(ended)
+                flags(uid, found)
+                deactivated(uid, "gone" if found is None else "disabled", row[0] if row else str(uid))
+            else:
+                upsert_user(conn, found, groups=[])
+                flags(uid, found)
+                set_roles(conn, uid, [])
+                res.sessions_ended += len(end_sessions(conn, uid, "roles_lost", by=RECONCILE_BY))
+        detail: dict[str, Any] = dict(res.summary())
+        if run_id is not None:
+            detail["run_id"] = run_id
+        audit.append(conn, action="ad.refresh", detail=detail, ip=ip, **who)
+    log.info("AD reconcile: %s", res.summary())
     return res
+
+
+reconcile = refresh_from_ad

server/packetstation/auth/ldap.py

@@ -32,6 +32,8 @@ USER_ATTRS = (
 ACCOUNTDISABLE = 2
 INVALID_CREDENTIALS = 49
 PAGE_SIZE = 500
+GUID_BATCH = 50  # objectGUID terms in one OR filter
+SEARCH_GROUPS_MAX = 100
 
 
 class LdapUnavailable(Exception):
@@ -60,6 +62,10 @@ class DirectoryProtocol(Protocol):
     def groups_of(self, dn: str) -> list[str]: ...
     def list_groups(self) -> list[str]: ...
     def members_of(self, group_cn: str) -> list[AdUser]: ...
+    def list_groups_detail(self) -> list[tuple[str, str]]: ...
+    def members_of_dn(self, group_dn: str) -> list[AdUser]: ...
+    def search_groups(self, q: str, limit: int = 25) -> list[dict[str, str]]: ...
+    def users_by_guid(self, guids: list[uuid.UUID]) -> dict[uuid.UUID, AdUser | None]: ...
 
 
 # ---------------------------------------------------------------- attribute helpers
@@ -287,6 +293,7 @@ class Directory:
         return sorted({cn for e in found if (cn := _text(e.get("attributes") or {}, "cn"))})
 
     def members_of(self, group_cn: str) -> list[AdUser]:
+        """Every account in the group under LDAP_GROUP_BASE_DN, nested included (IN_CHAIN, paged)."""
         from ldap3.utils.conv import escape_filter_chars
 
         with self._service() as conn:
@@ -299,6 +306,74 @@ class Directory:
             if not groups:
                 return []
             gdn = _text(groups[0].get("attributes") or {}, "distinguishedName") or str(groups[0].get("dn") or "")
-            flt = f"(&(objectClass=user)(objectCategory=person)(memberOf:{IN_CHAIN}:={escape_filter_chars(gdn)}))"
-            found = self._search(conn, self.settings.ldap_base_dn, flt, USER_ATTRS)
+            return self._members(conn, gdn)
+
+    def _members(self, conn: Any, group_dn: str) -> list[AdUser]:
+        from ldap3.utils.conv import escape_filter_chars
+
+        flt = f"(&(objectClass=user)(objectCategory=person)(memberOf:{IN_CHAIN}:={escape_filter_chars(group_dn)}))"
+        found = self._search(conn, self.settings.ldap_base_dn, flt, USER_ATTRS)
         return [user_from_entry(e) for e in found]
+
+    def list_groups_detail(self) -> list[tuple[str, str]]:
+        """(cn, dn) of every group in LDAP_GROUP_BASE_DN, sorted by cn."""
+        with self._service() as conn:
+            found = self._search(
+                conn, self.settings.ldap_group_base_dn, "(objectClass=group)", ["cn", "distinguishedName"]
+            )
+        out: dict[str, tuple[str, str]] = {}
+        for e in found:
+            attrs = e.get("attributes") or {}
+            cn = _text(attrs, "cn")
+            if cn:
+                out[cn.lower()] = (cn, _text(attrs, "distinguishedName") or str(e.get("dn") or ""))
+        return sorted(out.values(), key=lambda t: t[0].lower())
+
+    def members_of_dn(self, group_dn: str) -> list[AdUser]:
+        """Every account in the group by DN, nested included, disabled accounts too. Paged at 500."""
+        with self._service() as conn:
+            return self._members(conn, group_dn)
+
+    def search_groups(self, q: str, limit: int = 25) -> list[dict[str, str]]:
+        """Groups anywhere under LDAP_BASE_DN whose name or description contains q: [{dn, name, description}]."""
+        from ldap3.utils.conv import escape_filter_chars
+
+        q = q.strip()
+        if not q:
+            return []
+        limit = max(1, min(int(limit), SEARCH_GROUPS_MAX))
+        term = escape_filter_chars(q)
+        flt = f"(&(objectClass=group)(|(cn=*{term}*)(description=*{term}*)))"
+        with self._service() as conn:
+            found = self._search(conn, self.settings.ldap_base_dn, flt, ["cn", "distinguishedName", "description"])
+        out = []
+        for e in found:
+            attrs = e.get("attributes") or {}
+            name = _text(attrs, "cn")
+            if not name:
+                continue
+            out.append(
+                {
+                    "dn": _text(attrs, "distinguishedName") or str(e.get("dn") or ""),
+                    "name": name,
+                    "description": _text(attrs, "description") or "",
+                }
+            )
+        out.sort(key=lambda g: (not g["name"].lower().startswith(q.lower()), g["name"].lower()))
+        return out[:limit]
+
+    def users_by_guid(self, guids: list[uuid.UUID]) -> dict[uuid.UUID, AdUser | None]:
+        """The accounts for these objectGUIDs, read in batches. None for a GUID AD no longer has."""
+        wanted = list(dict.fromkeys(guids))
+        out: dict[uuid.UUID, AdUser | None] = dict.fromkeys(wanted)
+        if not wanted:
+            return out
+        with self._service() as conn:
+            for i in range(0, len(wanted), GUID_BATCH):
+                part = wanted[i : i + GUID_BATCH]
+                flt = "(|" + "".join(guid_filter(g) for g in part) + ")"
+                for e in self._search(conn, self.settings.ldap_base_dn, flt, USER_ATTRS):
+                    u = user_from_entry(e)
+                    if u.guid in out:
+                        out[u.guid] = u
+        return out

server/packetstation/auth/pin.py

@@ -101,6 +101,9 @@ def register_failure(
             detail={"failures": int(row[0]), "where": where},
             ip=ip,
         )
+        from ..domain import security_alerts
+
+        security_alerts.pin_disabled(conn, user_id, int(row[0]), where)
     return True
 
 

server/packetstation/auth/ratelimit.py

@@ -14,6 +14,7 @@ from datetime import datetime, timedelta
 from typing import Any
 
 from ..settings import Settings
+from . import secsettings
 
 RATE_LIMITED = "rate_limited"
 KINDS = ("password", "pin", "unlock", "signature", "pin_set")
@@ -62,7 +63,10 @@ def check(
     ip: str | None,
     now: datetime | None = None,
 ) -> int | None:
-    """Seconds until another attempt is allowed, or None when allowed. Failures of all `accounts` count together."""
+    """Seconds until another attempt is allowed, or None when allowed. Failures of all `accounts` count together.
+
+    The limits are the effective security settings (the Admin's values over the environment defaults)."""
+    settings = secsettings.overlay(conn, settings)
     now = now or db_now(conn)
     window = timedelta(minutes=settings.signin_account_window_minutes)
     cooldown = timedelta(minutes=settings.signin_account_cooldown_minutes)

server/packetstation/auth/secsettings.py

@@ -0,0 +1,252 @@
+"""Security settings: the `security` settings row laid over the environment defaults.
+
+The Admin Security tab edits the row. Every reader of these values (sessions, idle lock, roles refresh, sign-in
+rate limits) goes through `effective()` or `overlay()`. The stored row is cached in the process for at most
+CACHE_SECONDS; a save clears the cache at once and NOTIFYs `ps_settings` so the worker picks it up.
+A stored value out of its bounds (an older bound, a hand edit) is ignored and the default applies.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .. import audit
+from ..settings import Settings
+
+log = logging.getLogger(__name__)
+
+KEY = "security"
+SETTINGS_CHANNEL = "ps_settings"
+CACHE_SECONDS = 30.0
+
+# key: (min, max). The names are the Settings field names where one exists.
+BOUNDS: dict[str, tuple[int, int]] = {
+    "session_idle_minutes": (1, 60),
+    "session_lock_minutes": (1, 30),
+    "session_desk_minutes": (5, 240),
+    "session_password_hours": (1, 24),
+    "session_pin_hours": (1, 24),
+    "roles_refresh_minutes": (5, 60),
+    "signin_account_limit": (1, 15),  # always under AD's lockout at 20
+    "signin_account_window_minutes": (1, 60),
+    "signin_account_cooldown_minutes": (0, 120),
+    "signin_ip_limit": (5, 200),
+    "signin_ip_window_minutes": (1, 60),
+    "signin_burst_threshold": (5, 200),
+}
+KEYS = tuple(BOUNDS)
+# Keys that are not a Settings field: their default lives here.
+EXTRA_DEFAULTS: dict[str, int] = {"signin_burst_threshold": 20}
+BURST_WINDOW_MINUTES = 5
+
+
+class SecSettingsError(ValueError):
+    """Values the bounds refuse. `errors` holds one short line per problem."""
+
+    def __init__(self, errors: list[str]):
+        super().__init__("; ".join(errors))
+        self.errors = errors
+
+
+class VersionConflict(Exception):
+    def __init__(self, version: int):
+        super().__init__(f"version is now {version}")
+        self.version = version
+
+
+@dataclass(frozen=True)
+class SecSettings:
+    """The effective values. Every key is an attribute; `settings` is the Settings copy with them applied."""
+
+    session_idle_minutes: int
+    session_lock_minutes: int
+    session_desk_minutes: int
+    session_password_hours: int
+    session_pin_hours: int
+    roles_refresh_minutes: int
+    signin_account_limit: int
+    signin_account_window_minutes: int
+    signin_account_cooldown_minutes: int
+    signin_ip_limit: int
+    signin_ip_window_minutes: int
+    signin_burst_threshold: int
+    version: int = 0
+    settings: Settings | None = field(default=None, compare=False, repr=False)
+
+    def values(self) -> dict[str, int]:
+        return {k: int(getattr(self, k)) for k in KEYS}
+
+
+def defaults(settings: Settings) -> dict[str, int]:
+    """The environment defaults (Settings), plus the defaults of the keys Settings does not have."""
+    out: dict[str, int] = {}
+    for k in KEYS:
+        out[k] = int(getattr(settings, k)) if k in Settings.model_fields else EXTRA_DEFAULTS[k]
+    return out
+
+
+def _in_bounds(key: str, value: Any) -> bool:
+    if isinstance(value, bool) or not isinstance(value, int):
+        return False
+    lo, hi = BOUNDS[key]
+    return lo <= value <= hi
+
+
+def stored_values(value: Any) -> dict[str, int]:
+    """The good keys of a stored row value; anything unknown, not an integer, or out of bounds is dropped."""
+    if not isinstance(value, dict):
+        return {}
+    return {k: int(v) for k, v in value.items() if k in BOUNDS and _in_bounds(k, v)}
+
+
+# ---------------------------------------------------------------- the row and its cache
+
+_lock = threading.Lock()
+# (host, port, database) -> (loaded monotonic time, stored values, version)
+_rows: dict[tuple[str, int, str], tuple[float, dict[str, int], int]] = {}
+# (id(base settings), version, values) -> (base, overlaid Settings); keeps base alive so its id is not reused
+_overlays: OrderedDict[tuple[int, int, tuple[tuple[str, int], ...]], tuple[Settings, Settings]] = OrderedDict()
+OVERLAYS_MAX = 32
+
+
+def _db_key(settings: Settings) -> tuple[str, int, str]:
+    return (settings.pg_host, int(settings.pg_port), settings.pg_database)
+
+
+def clear_cache() -> None:
+    with _lock:
+        _rows.clear()
+        _overlays.clear()
+
+
+def load_versioned(conn: Any) -> tuple[dict[str, int], int, Any, Any]:
+    """(stored values, version, updated_at, updated_by) read now; version 0 when the row is missing."""
+    row = conn.execute("SELECT value, version, updated_at, updated_by FROM settings WHERE key = %s", (KEY,)).fetchone()
+    if row is None:
+        return {}, 0, None, None
+    return stored_values(row[0]), int(row[1]), row[2], row[3]
+
+
+def _stored(conn: Any, settings: Settings) -> tuple[dict[str, int], int]:
+    key = _db_key(settings)
+    now = time.monotonic()
+    with _lock:
+        hit = _rows.get(key)
+        if hit is not None and now - hit[0] < CACHE_SECONDS:
+            return hit[1], hit[2]
+    values, version, _, _ = load_versioned(conn)
+    with _lock:
+        _rows[key] = (now, values, version)
+    return values, version
+
+
+def _is_overlay(settings: Settings) -> bool:
+    with _lock:
+        return any(v[1] is settings for v in _overlays.values())
+
+
+def _overlay_settings(base: Settings, values: dict[str, int], version: int) -> Settings:
+    upd = {k: v for k, v in values.items() if k in Settings.model_fields}
+    if not upd:
+        return base
+    key = (id(base), version, tuple(sorted(upd.items())))
+    with _lock:
+        hit = _overlays.get(key)
+        if hit is not None and hit[0] is base:
+            _overlays.move_to_end(key)
+            return hit[1]
+    out = base.model_copy(update=upd)
+    with _lock:
+        _overlays[key] = (base, out)
+        while len(_overlays) > OVERLAYS_MAX:
+            _overlays.popitem(last=False)
+    return out
+
+
+def effective(conn: Any, settings: Settings) -> SecSettings:
+    """The effective security settings: the stored row over the environment defaults."""
+    values, version = _stored(conn, settings)
+    merged = {**defaults(settings), **values}
+    over = settings if _is_overlay(settings) else _overlay_settings(settings, values, version)
+    return SecSettings(**merged, version=version, settings=over)
+
+
+def overlay(conn: Any, settings: Settings) -> Settings:
+    """`settings` with the stored security values applied. Passing an overlaid copy returns it as it is."""
+    if _is_overlay(settings):
+        return settings
+    values, version = _stored(conn, settings)
+    return _overlay_settings(settings, values, version)
+
+
+def burst_threshold(conn: Any, settings: Settings) -> int:
+    return effective(conn, settings).signin_burst_threshold
+
+
+# ---------------------------------------------------------------- saving
+
+
+def validate(values: Any) -> dict[str, int]:
+    """A partial set of values, checked against the bounds. Raises SecSettingsError."""
+    if not isinstance(values, dict):
+        raise SecSettingsError(["values must be an object"])
+    errors: list[str] = []
+    out: dict[str, int] = {}
+    for k, v in values.items():
+        if k not in BOUNDS:
+            errors.append(f"{k}: not a security setting")
+            continue
+        lo, hi = BOUNDS[k]
+        if not _in_bounds(k, v):
+            errors.append(f"{k}: a whole number from {lo} to {hi}")
+            continue
+        out[k] = int(v)
+    if errors:
+        raise SecSettingsError(errors)
+    return out
+
+
+def save(conn: Any, settings: Settings, *, who: Any, values: Any, version: int) -> tuple[dict[str, int], int]:
+    """Compare-and-set a partial change. Returns (stored values, new version). Raises SecSettingsError or
+    VersionConflict. Audited `admin.security_settings_updated` with before and after (effective values)."""
+    patch = validate(values)
+    with conn.transaction():
+        row = conn.execute("SELECT value, version FROM settings WHERE key = %s FOR UPDATE", (KEY,)).fetchone()
+        current = stored_values(row[0]) if row else {}
+        cur_version = int(row[1]) if row else 0
+        if int(version) != cur_version:
+            raise VersionConflict(cur_version)
+        base = defaults(settings)
+        before = {**base, **current}
+        new = {**current, **patch}
+        after = {**base, **new}
+        changed = {k: {"from": before[k], "to": after[k]} for k in KEYS if before[k] != after[k]}
+        new_version = conn.execute(
+            "INSERT INTO settings (key, value, version, updated_at, updated_by) VALUES (%s, %s, 1, now(), %s)"
+            " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
+            " updated_at = now(), updated_by = EXCLUDED.updated_by RETURNING version",
+            (KEY, Jsonb(new), who.user_id),
+        ).fetchone()[0]
+        audit.append(
+            conn,
+            action="admin.security_settings_updated",
+            user_id=who.user_id,
+            session_id=getattr(who, "session_id", None),
+            role_snapshot=getattr(who, "role_snapshot", None),
+            subject_type="settings",
+            subject_id=KEY,
+            detail={"before": before, "after": after, "changed": changed},
+            ip=getattr(who, "ip", None),
+        )
+        payload = json.dumps({"key": KEY, "version": int(new_version)}, separators=(",", ":"))
+        conn.execute("SELECT ps_notify(%s, %s)", (SETTINGS_CHANNEL, payload))
+    clear_cache()
+    return new, int(new_version)

server/packetstation/auth/sessions.py

@@ -14,6 +14,7 @@ from psycopg.types.json import Jsonb
 
 from .. import events
 from ..settings import Settings
+from . import secsettings
 
 SESSION_COOKIE = "ps_session"
 DEVICE_COOKIE = "ps_device"
@@ -58,6 +59,7 @@ def create(
 ) -> tuple[str, dict[str, Any]]:
     """Insert a session row; returns (cookie token, row). The token itself is never stored."""
     token = new_token()
+    settings = secsettings.overlay(conn, settings)
     hours = settings.session_pin_hours if kind == "pin" else settings.session_password_hours
     with conn.cursor(row_factory=dict_row) as cur:
         row = cur.execute(

server/packetstation/auth/signin.py

@@ -14,6 +14,7 @@ from psycopg.rows import dict_row
 
 from .. import audit
 from ..api.errors import ApiError
+from ..domain import security_alerts
 from ..settings import Settings
 from . import devices, ratelimit, sessions
 from . import directory as userdir
@@ -91,6 +92,7 @@ def rate_gate(
 def _fail(
     conn: Any,
     *,
+    settings: Settings,
     account: str,
     ip: str | None,
     kind: str,
@@ -111,6 +113,7 @@ def _fail(
                 detail={"reason": reason, "account": account, "kind": kind},
                 ip=ip,
             )
+    security_alerts.after_failure(conn, settings, account=account, ip=ip, kind=kind, user_id=user_id)
     raise ApiError(status, code, detail)
 
 
@@ -187,6 +190,7 @@ def password_signin(
         # Refused before LDAP: an empty password would be an anonymous bind
         _fail(
             conn,
+            settings=settings,
             account=account or "(empty)",
             ip=ip,
             kind="password",
@@ -203,6 +207,7 @@ def password_signin(
     if u is None:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="password",
@@ -230,6 +235,7 @@ def password_signin(
     if not ok:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="password",
@@ -242,6 +248,7 @@ def password_signin(
     if u.disabled:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="password",
@@ -259,6 +266,7 @@ def password_signin(
     if not roles:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="password",
@@ -302,6 +310,7 @@ def pin_signin(
     def fail(reason: str, user_id: Any = None, *, audited: bool = False) -> NoReturn:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="pin",
@@ -338,6 +347,7 @@ def pin_signin(
         with conn.transaction():
             ratelimit.record(conn, account=account, ip=ip, kind="pin", ok=False, reason="bad_pin", user_id=uid)
             pins.register_failure(conn, settings, uid, ip=ip, where="signin")
+        security_alerts.after_failure(conn, settings, account=account, ip=ip, kind="pin", user_id=uid)
         raise ApiError(401, "signin_failed", PIN_FAILED)
     try:
         u = directory.get_user_by_guid(uid)
@@ -354,6 +364,7 @@ def pin_signin(
     if not roles:
         _fail(
             conn,
+            settings=settings,
             account=account,
             ip=ip,
             kind="pin",
@@ -423,6 +434,7 @@ def set_pin(
             user_id=actor.user_id,
             session_id=actor.session_id,
         )
+        security_alerts.after_failure(conn, settings, account=account, ip=ip, kind="pin_set", user_id=actor.user_id)
         raise ApiError(403, "bad_password", "The password is wrong.")
     with conn.transaction():
         pins.set_pin(conn, actor.user_id, new_pin)

server/packetstation/auth/stepup.py

@@ -11,6 +11,7 @@ from typing import Any
 
 from .. import audit
 from ..api.errors import ApiError
+from ..domain import security_alerts
 from ..settings import Settings
 from . import pin as pins
 from . import ratelimit, sessions
@@ -121,8 +122,10 @@ def verify(
             (actor.session_id,),
         ).fetchone()
         failures = int(row[0]) if row else settings.stepup_max_failures
+    security_alerts.after_failure(conn, settings, account=account, ip=ip, kind=kind, user_id=actor.user_id)
     if failures >= settings.stepup_max_failures:
         end_session(conn, actor, "stepup_failures", ip)
+        security_alerts.stepup_session_ended(conn, actor.user_id, failures, kind, ip)
         raise ApiError(
             401,
             "session_ended",

server/packetstation/domain/assets.py

@@ -593,33 +593,45 @@ def open_quota_alert(conn: Any) -> dict[str, Any] | None:
 def check_quota(conn: Any, settings: Settings) -> dict[str, Any]:
     """Open or resolve the storage alert. Returns the usage with `alert`: "opened", "resolved" or None.
 
-    Over the threshold with no open `assets_quota` row: insert one (severity warning, subject assets, detail the
-    usage) and audit `storage.quota_warning`. Under it with one open: set its resolved_at.
+    Over the threshold with no open `assets_quota` event: raise one through notify.raise_event (a standing
+    condition, subject assets, detail the usage) and audit `storage.quota_warning`. Under it with one open:
+    notify.resolve_event closes it and queues the recovery notice.
     """
+    from ..notify import raise_event, resolve_event
+
     usage = usage_for(conn, settings)
     change: str | None = None
+    warn_pct = int(settings.assets_quota_warn_pct)
     with conn.transaction():
         open_row = open_quota_alert(conn)
         if usage["warn"] and open_row is None:
-            r = conn.execute(
-                "INSERT INTO notification_events (kind, severity, subject, detail) VALUES (%s, 'warning', %s, %s)"
-                " ON CONFLICT (kind, coalesce(subject, '')) WHERE resolved_at IS NULL DO NOTHING RETURNING id",
-                (QUOTA_KIND, QUOTA_SUBJECT, Jsonb(usage)),
-            ).fetchone()
-            if r is not None:
+            new_id = raise_event(
+                conn,
+                QUOTA_KIND,
+                subject=QUOTA_SUBJECT,
+                title=f"Assets storage is {usage['pct']}% full",
+                body=f"The assets storage is {usage['pct']}% full. The warning is at {warn_pct}%."
+                f" Used {usage['used']} of {usage['total']} bytes.",
+                detail=usage,
+                condition=True,
+            )
+            if new_id is not None:
                 change = "opened"
                 audit.append(
                     conn,
                     action="storage.quota_warning",
                     subject_type="notification",
-                    subject_id=str(r[0]),
-                    detail={**usage, "warn_pct": int(settings.assets_quota_warn_pct)},
-                )
-                log.warning(
-                    "assets storage is %.1f%% full (warning at %s%%)", usage["pct"], settings.assets_quota_warn_pct
+                    subject_id=str(new_id),
+                    detail={**usage, "warn_pct": warn_pct},
                 )
+                log.warning("assets storage is %.1f%% full (warning at %s%%)", usage["pct"], warn_pct)
         elif not usage["warn"] and open_row is not None:
-            conn.execute("UPDATE notification_events SET resolved_at = now() WHERE id = %s", (open_row["id"],))
+            resolve_event(
+                conn,
+                QUOTA_KIND,
+                subject=QUOTA_SUBJECT,
+                body=f"The assets storage is back under the warning: {usage['pct']}% full.",
+            )
             change = "resolved"
             log.info("assets storage is back under the warning: %.1f%%", usage["pct"])
     return {**usage, "alert": change}

server/packetstation/domain/departments.py

@@ -0,0 +1,266 @@
+"""Departments and their stations, as the Admin edits them.
+
+Code and slug never change (the group names derive from them; a trigger refuses it too). Stations are never deleted,
+because station limits and packets point at them: `active: false` hides one. A new department is a shop department
+with the three shop groups IT must create in the PacketStation OU. The support departments map to fixed roles, so a
+new one would have no group and no role; it is refused.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from .. import audit
+from ..api.errors import ApiError
+from ..auth import groups as groupmap
+
+RX_CODE = re.compile(r"^[0-9A-Z]{2,4}$")
+RX_SLUG = re.compile(r"^[A-Za-z][A-Za-z0-9]{1,30}$")
+NAME_MAX = 80
+STATION_NAME_MAX = 60
+STATIONS_MAX = 60
+
+
+def clean_name(value: Any, what: str, limit: int) -> str:
+    text = " ".join(str(value or "").split())
+    if not text:
+        raise ApiError(422, "invalid_request", f"Give the {what} a name.")
+    if len(text) > limit:
+        raise ApiError(422, "invalid_request", f"The {what} name is longer than {limit} characters.")
+    return text
+
+
+def group_names_for(code: str, slug: str, kind: str, supervisor_group: bool) -> list[tuple[str, str]]:
+    """(group name, role) for one department, as the group map builds them."""
+    if kind == "shop":
+        return [(f"PS-{code}-{slug}-{lv.capitalize()}", f"shop:{code}:{lv}") for lv in groupmap.SHOP_LEVELS]
+    kinds = groupmap.OFFICE_KINDS.get(code)
+    if kinds is None:
+        return []
+    member, sup = kinds
+    out = [(f"PS-{slug}", member)]
+    if supervisor_group and sup:
+        out.append((f"PS-{slug}-Supervisor", sup))
+    return out
+
+
+def _stations(conn: Any, code: str | None = None) -> dict[str, list[dict[str, Any]]]:
+    sql = "SELECT dept_code, id, name, sort, active FROM stations"
+    params: tuple[Any, ...] = ()
+    if code is not None:
+        sql += " WHERE dept_code = %s"
+        params = (code,)
+    out: dict[str, list[dict[str, Any]]] = {}
+    for dept, sid, name, sort, active in conn.execute(sql + " ORDER BY dept_code, sort, name, id", params):
+        out.setdefault(dept, []).append({"id": int(sid), "name": name, "sort": int(sort), "active": bool(active)})
+    return out
+
+
+def _out(r: dict[str, Any], stations: list[dict[str, Any]], in_ou: set[str] | None) -> dict[str, Any]:
+    groups = [
+        {"name": name, "role": role, "in_ou": None if in_ou is None else name.lower() in in_ou}
+        for name, role in group_names_for(r["code"], r["slug"], r["kind"], bool(r["supervisor_group"]))
+    ]
+    return {
+        "code": r["code"],
+        "slug": r["slug"],
+        "name": r["name"],
+        "kind": r["kind"],
+        "sort": int(r["sort"]),
+        "active": bool(r["active"]),
+        "weld": bool(r["weld"]),
+        "supervisor_group": bool(r["supervisor_group"]),
+        "created_at": r["created_at"],
+        "groups": groups,
+        "stations": stations,
+    }
+
+
+SELECT = (
+    "SELECT code, slug, name, kind::text AS kind, sort, active, weld, supervisor_group, created_at FROM departments"
+)
+
+
+def list_all(conn: Any, in_ou: list[str] | None) -> list[dict[str, Any]]:
+    """Every department, active or not, in sort order. `in_ou` is the group names in the OU (None: unknown)."""
+    lower = {n.lower() for n in in_ou} if in_ou is not None else None
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(SELECT + " ORDER BY sort, code").fetchall()
+    st = _stations(conn)
+    return [_out(r, st.get(r["code"], []), lower) for r in rows]
+
+
+def get(conn: Any, code: str, in_ou: list[str] | None = None) -> dict[str, Any] | None:
+    lower = {n.lower() for n in in_ou} if in_ou is not None else None
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(SELECT + " WHERE code = %s", (code,)).fetchone()
+    if r is None:
+        return None
+    return _out(r, _stations(conn, code).get(code, []), lower)
+
+
+def _snapshot(d: dict[str, Any]) -> dict[str, Any]:
+    return {k: d[k] for k in ("name", "active", "stations")}
+
+
+def update(
+    conn: Any,
+    who: Any,
+    code: str,
+    *,
+    name: str | None,
+    active: bool | None,
+    stations: list[dict[str, Any]] | None,
+) -> dict[str, Any]:
+    """Change the name, active and stations of one department. Audited `admin.department_updated`."""
+    with conn.transaction():
+        locked = conn.execute("SELECT code FROM departments WHERE code = %s FOR UPDATE", (code,)).fetchone()
+        if locked is None:
+            raise ApiError(404, "not_found", "No such department.")
+        before = get(conn, code)
+        assert before is not None
+        if name is not None:
+            conn.execute(
+                "UPDATE departments SET name = %s WHERE code = %s", (clean_name(name, "department", NAME_MAX), code)
+            )
+        if active is not None:
+            conn.execute("UPDATE departments SET active = %s WHERE code = %s", (bool(active), code))
+        if stations is not None:
+            _apply_stations(conn, code, before["stations"], stations)
+        after = get(conn, code)
+        assert after is not None
+        if _snapshot(before) != _snapshot(after):
+            audit.append(
+                conn,
+                action="admin.department_updated",
+                user_id=who.user_id,
+                session_id=who.session_id,
+                role_snapshot=who.role_snapshot,
+                subject_type="department",
+                subject_id=code,
+                detail={"before": _snapshot(before), "after": _snapshot(after)},
+                ip=who.ip,
+            )
+    return after
+
+
+def _apply_stations(conn: Any, code: str, current: list[dict[str, Any]], wanted: list[dict[str, Any]]) -> None:
+    by_id = {s["id"]: s for s in current}
+    final: dict[int, dict[str, Any]] = {s["id"]: dict(s) for s in current}
+    new: list[dict[str, Any]] = []
+    seen_ids: set[int] = set()
+    for w in wanted:
+        sid = w.get("id")
+        entry = {
+            "name": clean_name(w.get("name"), "station", STATION_NAME_MAX),
+            "sort": int(w.get("sort") or 0),
+            "active": bool(w.get("active", True)),
+        }
+        if sid is None:
+            new.append(entry)
+            continue
+        if sid not in by_id:
+            raise ApiError(422, "invalid_request", f"Station {sid} is not in department {code}.")
+        if sid in seen_ids:
+            raise ApiError(422, "invalid_request", f"Station {sid} is listed twice.")
+        seen_ids.add(sid)
+        final[sid].update(entry)
+    names = [s["name"].lower() for s in final.values()] + [s["name"].lower() for s in new]
+    dupes = sorted({n for n in names if names.count(n) > 1})
+    if dupes:
+        raise ApiError(422, "duplicate_station", f"Two stations in {code} have the same name: {', '.join(dupes)}.")
+    if len(final) + len(new) > STATIONS_MAX:
+        raise ApiError(422, "invalid_request", f"A department has at most {STATIONS_MAX} stations.")
+    renamed = [sid for sid, s in final.items() if s["name"] != by_id[sid]["name"]]
+    # Names are unique per department: park the renamed ones first so two stations can swap names.
+    for sid in renamed:
+        conn.execute("UPDATE stations SET name = %s WHERE id = %s", (f"~{sid}~renaming", sid))
+    for sid, s in final.items():
+        if s != by_id[sid]:
+            conn.execute(
+                "UPDATE stations SET name = %s, sort = %s, active = %s WHERE id = %s",
+                (s["name"], s["sort"], s["active"], sid),
+            )
+    for s in new:
+        conn.execute(
+            "INSERT INTO stations (dept_code, name, sort, active) VALUES (%s, %s, %s, %s)",
+            (code, s["name"], s["sort"], s["active"]),
+        )
+
+
+def add(
+    conn: Any,
+    who: Any,
+    *,
+    code: str,
+    slug: str,
+    name: str,
+    kind: str,
+    weld: bool,
+    supervisor_group: bool,
+    stations: list[str],
+) -> tuple[dict[str, Any], list[str]]:
+    """Add a shop department. Returns (department, group names IT must create). Audited `admin.department_added`."""
+    code, slug = code.strip(), slug.strip()
+    if not RX_CODE.match(code):
+        raise ApiError(422, "invalid_request", "The code is 2 to 4 capital letters or digits, like 15.")
+    if not RX_SLUG.match(slug):
+        raise ApiError(422, "invalid_request", "The short name is a letter, then letters or digits, no spaces.")
+    if kind != "shop":
+        raise ApiError(
+            422,
+            "support_fixed",
+            "Only a shop department can be added. The support departments map to fixed roles.",
+        )
+    title = clean_name(name, "department", NAME_MAX)
+    names: list[str] = []
+    for s in stations:
+        n = clean_name(s, "station", STATION_NAME_MAX)
+        if n.lower() in (x.lower() for x in names):
+            raise ApiError(422, "duplicate_station", f"Two stations have the same name: {n}.")
+        names.append(n)
+    if len(names) > STATIONS_MAX:
+        raise ApiError(422, "invalid_request", f"A department has at most {STATIONS_MAX} stations.")
+    with conn.transaction():
+        conn.execute("LOCK TABLE departments IN SHARE ROW EXCLUSIVE MODE")
+        if conn.execute("SELECT 1 FROM departments WHERE upper(code) = upper(%s)", (code,)).fetchone():
+            raise ApiError(409, "code_taken", f"Department code {code} is taken.")
+        if conn.execute("SELECT 1 FROM departments WHERE lower(slug) = lower(%s)", (slug,)).fetchone():
+            raise ApiError(409, "slug_taken", f"The short name {slug} is taken.")
+        sort = int(conn.execute("SELECT coalesce(max(sort), 0) + 10 FROM departments").fetchone()[0])
+        conn.execute(
+            "INSERT INTO departments (code, slug, name, kind, sort, active, supervisor_group, weld, created_at,"
+            " created_by) VALUES (%s, %s, %s, %s::dept_kind, %s, true, %s, %s, now(), %s)",
+            (code, slug, title, kind, sort, bool(supervisor_group), bool(weld), who.user_id),
+        )
+        for i, n in enumerate(names, start=1):
+            conn.execute("INSERT INTO stations (dept_code, name, sort) VALUES (%s, %s, %s)", (code, n, i))
+        dept = get(conn, code)
+        assert dept is not None
+        groups = [g["name"] for g in dept["groups"]]
+        audit.append(
+            conn,
+            action="admin.department_added",
+            user_id=who.user_id,
+            session_id=who.session_id,
+            role_snapshot=who.role_snapshot,
+            subject_type="department",
+            subject_id=code,
+            detail={
+                "after": {
+                    "code": code,
+                    "slug": slug,
+                    "name": title,
+                    "kind": kind,
+                    "weld": bool(weld),
+                    "supervisor_group": bool(supervisor_group),
+                    "stations": names,
+                },
+                "groups_to_create": groups,
+            },
+            ip=who.ip,
+        )
+    return dept, groups

server/packetstation/domain/itp.py

@@ -30,6 +30,7 @@ from ..api import deps
 from ..api.errors import ApiError
 from ..auth import signatures
 from ..auth.signatures import Signature
+from ..notify import raise_event
 from ..policy.actions import SCOPED_LABELS
 from ..policy.engine import check_scoped
 from ..text import LIMITS, clean_opt, reason_ok
@@ -593,5 +594,38 @@ def _finish(w: ItpWrite) -> None:
     events.publish(conn, {"event": "changed", "kind": "itp", "unit_id": w.itp["unit_id"], "itp_id": w.id})
 
 
+def reject_dept(conn: Any, unit_id: Any, page: Any) -> str | None:
+    """The department of a rejected line: the one department among the unit's live packets that works to the
+    line's page, or None when there is none or more than one."""
+    codes = {
+        str(r[0])
+        for r in conn.execute(
+            "SELECT DISTINCT dept_code FROM packets WHERE unit_id = %s AND removed_at IS NULL", (unit_id,)
+        ).fetchall()
+    }
+    hits = sorted(c for c in codes if page and page in DEPT_PAGES.get(c, ()))
+    return hits[0] if len(hits) == 1 else None
+
+
+def notify_reject(w: ItpWrite, key: str, ncr: str | None, note: str | None) -> int | None:
+    """qc_reject for an ITP line reject, in the write's transaction (notify-send mails it)."""
+    item = next((i for i in items(w.itp.get("template") or {}) if str(i.get("n")) == key), {})
+    page = item.get("page")
+    dept = reject_dept(w.conn, w.itp["unit_id"], page)
+    who = (w.principal.user or {}).get("display_name") or ""
+    head = f"NCR {ncr}" if ncr else "NCR # pending"
+    lines = [str(item.get("t") or ""), head, note or "", f"Rejected by {who}." if who else ""]
+    where = f" ({PAGE_NAMES.get(str(page), page)})" if page else ""
+    return raise_event(
+        w.conn,
+        "qc_reject",
+        subject=f"itp:{w.id}:{key}",
+        title=f"ITP {w.itp.get('no') or w.id} line {key} rejected{where}",
+        body="\n".join(x for x in lines if x),
+        detail={"itp_id": w.id, "unit_id": w.itp["unit_id"], "line": key, "page": page, "ncr": ncr},
+        dept_code=dept,
+    )
+
+
 def label(action: policy.Action) -> str:
     return SCOPED_LABELS.get(action, str(action))

server/packetstation/domain/profiles.py

@@ -0,0 +1,370 @@
+"""People's Packet Station profiles: welder stamp, profile active, station limits. Edited one at a time or by a CSV
+import keyed on the employee ID.
+
+Roles never come from here; AD groups decide them. An inactive profile drops out of the welder picker and the
+worker filter. Employee IDs stay text (leading zeros kept).
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+from collections import Counter
+from collections.abc import Mapping
+from typing import Any
+
+from .. import audit
+from ..api.errors import ApiError
+
+RX_STAMP = re.compile(r"^[A-Za-z0-9-]{1,12}$")
+CSV_MAX_BYTES = 1_000_000
+CSV_MAX_ROWS = 2000
+COLUMNS = ("employee_id", "welder_stamp", "active", "stations")
+CLEAR = "-"
+TRUE_WORDS = ("yes", "true", "1")
+FALSE_WORDS = ("no", "false", "0")
+
+
+# ---------------------------------------------------------------- reading
+
+
+def station_limits(conn: Any, user_ids: list[Any]) -> dict[Any, dict[str, list[int]]]:
+    """user id -> {dept_code: [station ids]}."""
+    out: dict[Any, dict[str, list[int]]] = {}
+    if not user_ids:
+        return out
+    rows = conn.execute(
+        "SELECT l.user_id, s.dept_code, s.id FROM user_station_limits l JOIN stations s ON s.id = l.station_id"
+        " WHERE l.user_id = ANY(%s) ORDER BY s.dept_code, s.sort, s.id",
+        (user_ids,),
+    ).fetchall()
+    for uid, dept, sid in rows:
+        out.setdefault(uid, {}).setdefault(dept, []).append(int(sid))
+    return out
+
+
+def profile_of(conn: Any, user_id: Any) -> dict[str, Any] | None:
+    row = conn.execute("SELECT welder_stamp, profile_active FROM users WHERE id = %s", (user_id,)).fetchone()
+    if row is None:
+        return None
+    return {
+        "welder_stamp": row[0],
+        "profile_active": bool(row[1]),
+        "station_limits": station_limits(conn, [user_id]).get(user_id, {}),
+    }
+
+
+def _stamp_owner(conn: Any, stamp: str, user_id: Any) -> str | None:
+    row = conn.execute(
+        "SELECT display_name FROM users WHERE lower(welder_stamp) = lower(%s) AND id <> %s LIMIT 1", (stamp, user_id)
+    ).fetchone()
+    return str(row[0]) if row else None
+
+
+# ---------------------------------------------------------------- one person
+
+
+def _audit(conn: Any, who: Any, user_id: Any, before: dict[str, Any], after: dict[str, Any], source: str) -> None:
+    audit.append(
+        conn,
+        action="admin.profile_updated",
+        user_id=who.user_id,
+        session_id=who.session_id,
+        role_snapshot=who.role_snapshot,
+        subject_type="user",
+        subject_id=user_id,
+        detail={"before": before, "after": after, "source": source},
+        ip=who.ip,
+    )
+
+
+def _write(conn: Any, user_id: Any, stamp: Any, active: Any, limits: Mapping[str, list[int]] | None) -> None:
+    """Write the fields given; `stamp` and `active` use ... (Ellipsis) for unchanged. `limits` replaces the named
+    departments' limits; None leaves them."""
+    if stamp is not ...:
+        conn.execute("UPDATE users SET welder_stamp = %s WHERE id = %s", (stamp, user_id))
+    if active is not ...:
+        conn.execute("UPDATE users SET profile_active = %s WHERE id = %s", (bool(active), user_id))
+    for dept, ids in (limits or {}).items():
+        conn.execute(
+            "DELETE FROM user_station_limits l USING stations s WHERE s.id = l.station_id AND l.user_id = %s"
+            " AND s.dept_code = %s",
+            (user_id, dept),
+        )
+        for sid in ids:
+            conn.execute("INSERT INTO user_station_limits (user_id, station_id) VALUES (%s, %s)", (user_id, sid))
+
+
+def update(conn: Any, who: Any, user_id: Any, fields: Mapping[str, Any]) -> dict[str, Any]:
+    """PATCH a profile: welder_stamp (None clears), profile_active, station_limits {dept: [station id]} (a dept key
+    replaces that department's limits; an empty list clears them)."""
+    stamp: Any = ...
+    if "welder_stamp" in fields:
+        raw = fields["welder_stamp"]
+        stamp = None if raw is None or not str(raw).strip() else str(raw).strip()
+        if stamp is not None and not RX_STAMP.match(stamp):
+            raise ApiError(422, "bad_stamp", "A stamp is 1 to 12 letters, digits or dashes.")
+    active: Any = fields["profile_active"] if fields.get("profile_active") is not None else ...
+    limits: dict[str, list[int]] | None = None
+    if fields.get("station_limits") is not None:
+        limits = {}
+        for dept, ids in dict(fields["station_limits"]).items():
+            ok = {int(r[0]) for r in conn.execute("SELECT id FROM stations WHERE dept_code = %s", (dept,)).fetchall()}
+            if not conn.execute("SELECT 1 FROM departments WHERE code = %s", (dept,)).fetchone():
+                raise ApiError(422, "unknown_department", f"No department {dept}.")
+            want = sorted({int(x) for x in ids})
+            bad = [x for x in want if x not in ok]
+            if bad:
+                raise ApiError(422, "unknown_station", f"Station {bad[0]} is not in department {dept}.")
+            limits[str(dept)] = want
+    with conn.transaction():
+        if conn.execute("SELECT 1 FROM users WHERE id = %s FOR UPDATE", (user_id,)).fetchone() is None:
+            raise ApiError(404, "not_found", "No such user.")
+        if stamp is not ... and stamp is not None:
+            owner = _stamp_owner(conn, stamp, user_id)
+            if owner is not None:
+                raise ApiError(409, "stamp_taken", f"Stamp {stamp} belongs to {owner}.")
+        before = profile_of(conn, user_id)
+        assert before is not None
+        _write(conn, user_id, stamp, active, limits)
+        after = profile_of(conn, user_id)
+        assert after is not None
+        if before != after:
+            _audit(conn, who, user_id, before, after, "edit")
+    return after
+
+
+# ---------------------------------------------------------------- CSV import
+
+
+def _station_index(conn: Any) -> tuple[set[str], dict[tuple[str, str], int], dict[int, str]]:
+    depts = {r[0] for r in conn.execute("SELECT code FROM departments").fetchall()}
+    by_name: dict[tuple[str, str], int] = {}
+    label: dict[int, str] = {}
+    for sid, dept, name in conn.execute("SELECT id, dept_code, name FROM stations").fetchall():
+        by_name[(dept.lower(), " ".join(name.split()).lower())] = int(sid)
+        label[int(sid)] = f"{dept}:{name}"
+    return depts, by_name, label
+
+
+def _bool(text: str) -> bool | None:
+    t = text.strip().lower()
+    if t in TRUE_WORDS:
+        return True
+    if t in FALSE_WORDS:
+        return False
+    return None
+
+
+def _labels(ids: list[int], label: dict[int, str]) -> list[str]:
+    return sorted(label.get(i, str(i)) for i in ids)
+
+
+def parse_csv(text: str) -> tuple[list[str], list[tuple[int, dict[str, str]]]]:
+    """(header columns, [(line, {column: cell})]). Raises ApiError for a file that cannot be read at all."""
+    if len(text.encode("utf-8")) > CSV_MAX_BYTES:
+        raise ApiError(413, "too_large", "The file is larger than 1 MB.")
+    reader = csv.reader(io.StringIO(text.lstrip("")))
+    try:
+        header = next(reader, None)
+        if header is None:
+            raise ApiError(422, "bad_csv", "The file is empty.")
+        cols = [h.strip().lower() for h in header]
+        unknown = [c for c in cols if c and c not in COLUMNS]
+        if "employee_id" not in cols:
+            raise ApiError(422, "bad_csv", "The header row has no employee_id column.")
+        if unknown:
+            raise ApiError(422, "bad_csv", f"Unknown column: {', '.join(unknown)}.")
+        if len(set(c for c in cols if c)) != len([c for c in cols if c]):
+            raise ApiError(422, "bad_csv", "A column is named twice.")
+        rows: list[tuple[int, dict[str, str]]] = []
+        for cells in reader:
+            if not any(c.strip() for c in cells):
+                continue
+            if len(rows) >= CSV_MAX_ROWS:
+                raise ApiError(422, "bad_csv", f"The file has more than {CSV_MAX_ROWS} rows.")
+            row = {c: (cells[i].strip() if i < len(cells) else "") for i, c in enumerate(cols) if c}
+            rows.append((reader.line_num, row))
+    except csv.Error as e:
+        raise ApiError(422, "bad_csv", f"The file is not valid CSV: {e}.") from None
+    return cols, rows
+
+
+def import_profiles(conn: Any, who: Any, text: str, *, apply: bool) -> dict[str, Any]:
+    """Preview (apply false) or apply a profile CSV. With any error, apply is refused: 422 with the same body."""
+    _, parsed = parse_csv(text)
+    depts, by_name, label = _station_index(conn)
+    emps = [r.get("employee_id", "") for _, r in parsed]
+    users: dict[str, list[dict[str, Any]]] = {}
+    want = sorted({e for e in emps if e})
+    if want:
+        for uid, emp, name, stamp, active in conn.execute(
+            "SELECT id, emp_no, display_name, welder_stamp, profile_active FROM users"
+            " WHERE active AND emp_no = ANY(%s)",
+            (want,),
+        ).fetchall():
+            users.setdefault(emp, []).append(
+                {"id": uid, "display_name": name, "welder_stamp": stamp, "profile_active": bool(active)}
+            )
+    limits = station_limits(conn, [u["id"] for us in users.values() for u in us])
+    seen: Counter[str] = Counter()
+    first_line: dict[str, int] = {}
+    out: list[dict[str, Any]] = []
+    plans: list[tuple[dict[str, Any], dict[str, Any]]] = []  # (row out, plan)
+    for line, r in parsed:
+        emp = r.get("employee_id", "")
+        errors: list[str] = []
+        row_out: dict[str, Any] = {
+            "line": line,
+            "employee_id": emp,
+            "user_id": None,
+            "display_name": None,
+            "changes": {},
+            "errors": errors,
+        }
+        out.append(row_out)
+        if not emp:
+            errors.append("Employee ID is missing.")
+            continue
+        seen[emp] += 1
+        if seen[emp] > 1:
+            errors.append(f"Employee ID {emp} is repeated in the file (first on line {first_line[emp]}).")
+            continue
+        first_line[emp] = line
+        found = users.get(emp, [])
+        if not found:
+            errors.append(f"Unknown employee ID {emp}.")
+            continue
+        if len(found) > 1:
+            errors.append(f"Employee ID {emp} is on more than one user.")
+            continue
+        u = found[0]
+        row_out["user_id"] = str(u["id"])
+        row_out["display_name"] = u["display_name"]
+        plan: dict[str, Any] = {"user": u}
+        stamp_cell = r.get("welder_stamp", "")
+        if stamp_cell:
+            new_stamp = None if stamp_cell == CLEAR else stamp_cell
+            if new_stamp is not None and not RX_STAMP.match(new_stamp):
+                errors.append(f"Bad stamp {stamp_cell}: 1 to 12 letters, digits or dashes.")
+            elif new_stamp != u["welder_stamp"]:
+                plan["welder_stamp"] = new_stamp
+        active_cell = r.get("active", "")
+        if active_cell:
+            val = _bool(active_cell)
+            if val is None:
+                errors.append(f"Bad active value {active_cell}: use yes or no.")
+            elif val != u["profile_active"]:
+                plan["profile_active"] = val
+        st_cell = r.get("stations", "")
+        if st_cell:
+            current = sorted(i for ids in limits.get(u["id"], {}).values() for i in ids)
+            ids: list[int] = []
+            if st_cell != CLEAR:
+                for part in (p.strip() for p in st_cell.split(";")):
+                    if not part:
+                        continue
+                    dept, sep, name = part.partition(":")
+                    dept, name = dept.strip(), " ".join(name.split())
+                    if not sep or not dept or not name:
+                        errors.append(f"Bad station {part}: write <department code>:<station name>.")
+                    elif dept not in depts:
+                        errors.append(f"Unknown department {dept}.")
+                    elif (dept.lower(), name.lower()) not in by_name:
+                        errors.append(f"Unknown station {name} in department {dept}.")
+                    else:
+                        sid = by_name[(dept.lower(), name.lower())]
+                        if sid not in ids:
+                            ids.append(sid)
+            ids.sort()
+            if ids != current:
+                plan["stations"] = (current, ids)
+        plans.append((row_out, plan))
+
+    # Stamps must stay unique after the whole file applies.
+    final: dict[Any, str | None] = {}
+    for _, plan in plans:
+        if "welder_stamp" in plan:
+            final[plan["user"]["id"]] = plan["welder_stamp"]
+    claimed = [s for s in final.values() if s]
+    if claimed:
+        others = {
+            str(r[1]).lower(): (r[0], r[2])
+            for r in conn.execute(
+                "SELECT id, welder_stamp, display_name FROM users WHERE welder_stamp IS NOT NULL"
+                " AND lower(welder_stamp) = ANY(%s)",
+                ([s.lower() for s in claimed],),
+            ).fetchall()
+        }
+        names_by_id = {plan["user"]["id"]: plan["user"]["display_name"] for _, plan in plans}
+        taken_in_file: dict[str, Any] = {}
+        for row_out, plan in plans:
+            s = plan.get("welder_stamp")
+            if not s:
+                continue
+            uid = plan["user"]["id"]
+            holder = others.get(s.lower())
+            # The holder keeps the stamp unless the same file gives them another (or clears it)
+            freed = holder is not None and holder[0] in final and str(final[holder[0]] or "").lower() != s.lower()
+            if holder is not None and holder[0] != uid and not freed:
+                row_out["errors"].append(f"Stamp {s} is taken by {holder[1]}.")
+            elif s.lower() in taken_in_file:
+                row_out["errors"].append(f"Stamp {s} is taken by {names_by_id[taken_in_file[s.lower()]]}.")
+            else:
+                taken_in_file[s.lower()] = uid
+
+    for row_out, plan in plans:
+        u = plan["user"]
+        if "welder_stamp" in plan:
+            row_out["changes"]["welder_stamp"] = {"from": u["welder_stamp"], "to": plan["welder_stamp"]}
+        if "profile_active" in plan:
+            row_out["changes"]["profile_active"] = {"from": u["profile_active"], "to": plan["profile_active"]}
+        if "stations" in plan:
+            cur, ids = plan["stations"]
+            row_out["changes"]["station_limits"] = {"from": _labels(cur, label), "to": _labels(ids, label)}
+
+    n_err = sum(1 for r in out if r["errors"])
+    body: dict[str, Any] = {"rows": out, "ok": len(out) - n_err, "errors": n_err, "applied": False}
+    if not apply:
+        return body
+    if n_err:
+        raise ApiError(
+            422,
+            "import_errors",
+            f"{n_err} row{'s' if n_err != 1 else ''} with errors. Nothing was changed.",
+            extra=body,
+        )
+    changed = 0
+    with conn.transaction():
+        for row_out, plan in plans:
+            if not row_out["changes"]:
+                continue
+            uid = plan["user"]["id"]
+            conn.execute("SELECT 1 FROM users WHERE id = %s FOR UPDATE", (uid,))
+            before = profile_of(conn, uid)
+            stamp = plan.get("welder_stamp", ...)
+            active = plan.get("profile_active", ...)
+            lim: dict[str, list[int]] | None = None
+            if "stations" in plan:
+                conn.execute("DELETE FROM user_station_limits WHERE user_id = %s", (uid,))
+                for sid in plan["stations"][1]:
+                    conn.execute("INSERT INTO user_station_limits (user_id, station_id) VALUES (%s, %s)", (uid, sid))
+            _write(conn, uid, stamp, active, lim)
+            after = profile_of(conn, uid)
+            assert before is not None and after is not None
+            _audit(conn, who, uid, before, after, "import")
+            changed += 1
+        audit.append(
+            conn,
+            action="admin.profile_import",
+            user_id=who.user_id,
+            session_id=who.session_id,
+            role_snapshot=who.role_snapshot,
+            subject_type="users",
+            subject_id="import",
+            detail={"rows": len(out), "changed": changed, "unchanged": len(out) - changed},
+            ip=who.ip,
+        )
+    body["applied"] = True
+    return body

server/packetstation/domain/qc.py

@@ -10,6 +10,7 @@ from collections.abc import Mapping
 from typing import Any
 
 from .. import policy
+from ..notify import raise_event
 from ..text import LIMITS
 from . import itp, packets, qcdocs, search
 
@@ -65,6 +66,23 @@ def inspector_text(insp: Mapping[str, Any] | None) -> str:
     return f"{who} [{stamp}]" if stamp else who
 
 
+def _notify_reject(w: Any, why: str) -> None:
+    """qc_reject for the packet's department, in the write's transaction (notify-send mails it)."""
+    pkt = w.packet
+    dept = str(w.from_dept or pkt.get("dept_code") or "") or None
+    what = str(pkt.get("title") or pkt.get("drawing") or "")
+    who = (w.principal.user or {}).get("display_name") or ""
+    raise_event(
+        w.conn,
+        "qc_reject",
+        subject=f"packet:{w.id}",
+        title=f"Q.C. rejected packet {w.id}" + (f" ({what})" if what else "") + (f", dept {dept}" if dept else ""),
+        body="\n".join(x for x in (f"Rejected by {who}." if who else "", why) if x),
+        detail={"packet_id": w.id, "dept_code": dept, "by": str(w.user_id)},
+        dept_code=dept,
+    )
+
+
 def qc_texts(w: Any, a: policy.Action, to_dept: str) -> None:
     """The packet_log label and note of a Q.C. accept or reject. Called before the side effects close the inspection.
 
@@ -79,6 +97,7 @@ def qc_texts(w: Any, a: policy.Action, to_dept: str) -> None:
         if welds:
             parts.append("welds " + ", ".join(welds))
         w.log("Q.C. rejected → supervisor", " · ".join(x for x in parts if x) or None)
+        _notify_reject(w, " · ".join(x for x in parts if x))
         return
     if a is A.QC_ACCEPT_AND_ROUTE:
         labels = getattr(w.children, "dept_labels", None) or packets.dept_labels(w.conn)

server/packetstation/domain/search.py

@@ -369,7 +369,8 @@ JOIN stations st ON st.id = p.station_id WHERE p.removed_at IS NULL{scope} ORDER
 """
 FACET_WORKERS_SQL = """
 SELECT DISTINCT wu.id, wu.display_name FROM packets p JOIN units u ON u.id = p.unit_id
-JOIN users wu ON wu.id = p.welder_id WHERE p.removed_at IS NULL{scope} ORDER BY wu.display_name, wu.id
+JOIN users wu ON wu.id = p.welder_id AND wu.profile_active
+WHERE p.removed_at IS NULL{scope} ORDER BY wu.display_name, wu.id
 """
 
 

server/packetstation/domain/security_alerts.py

@@ -0,0 +1,321 @@
+"""Security alerts: raising them on a failed sign-in, a disabled PIN or an ended session, auditing policy denials,
+and the Admin Security tab's list of what happened.
+
+Raising goes through `notify.raise_event` (security_alert, signin_burst) inside the caller's database work. A
+failure here is logged and never breaks the sign-in or the request it rides on.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from datetime import timedelta
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from .. import audit
+from ..auth import ratelimit, secsettings
+from ..settings import Settings
+
+log = logging.getLogger(__name__)
+
+SECURITY_ALERT = "security_alert"
+SIGNIN_BURST = "signin_burst"
+ALERT_DEDUPE_MINUTES = 15
+BURST_DEDUPE_MINUTES = 30
+BURST_KINDS = ("password", "pin")
+PIN_FAILURES_MIN = 3
+ALERTS_MAX = 1000
+DENIED_DEDUPE_SECONDS = 60.0
+DENIED_SKIP_CODES = frozenset({"wrong_secret"})
+
+ALERT_KINDS = (
+    "lockout",
+    "pin_failures",
+    "denied",
+    "stepup_wrong_pin",
+    "admin_change",
+    "it_edit",
+    "integrity",
+    "signin_burst",
+)
+
+
+def notify(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.raise_event in a savepoint. A failure is logged and returns None."""
+    try:
+        from ..notify import raise_event
+
+        with conn.transaction():
+            result: int | None = raise_event(conn, event_type, **kw)
+            return result
+    except Exception:
+        log.exception("could not raise %s", event_type)
+        return None
+
+
+def _recent_audit(conn: Any, action: str, subject: str, minutes: int) -> bool:
+    row = conn.execute(
+        "SELECT 1 FROM audit_events WHERE action = %s AND subject_type = 'security' AND subject_id = %s"
+        " AND at > now() - make_interval(mins => %s::int) LIMIT 1",
+        (action, subject, max(1, minutes)),
+    ).fetchone()
+    return row is not None
+
+
+def _security_audit(
+    conn: Any, action: str, subject: str, detail: dict[str, Any], *, user_id: Any = None, ip: str | None = None
+) -> None:
+    audit.append(
+        conn, action=action, user_id=user_id, subject_type="security", subject_id=subject, detail=detail, ip=ip
+    )
+
+
+def after_failure(
+    conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any = None
+) -> None:
+    """Call after a failed attempt was recorded. Raises security_alert when the account's cooldown starts, and
+    signin_burst when the address limit trips or the failures across accounts reach the burst threshold."""
+    try:
+        with conn.transaction():
+            _after_failure(conn, settings, account=account, ip=ip, kind=kind, user_id=user_id)
+    except Exception:
+        log.exception("security alert check failed")
+
+
+def _after_failure(conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any) -> None:
+    sec = secsettings.effective(conn, settings)
+    s = sec.settings or settings
+    now = ratelimit.db_now(conn)
+    if account:
+        wait = ratelimit.check(conn, s, accounts=[account], ip=None, now=now)
+        subject = f"account:{account}"
+        if wait is not None and not _recent_audit(conn, "auth.lockout", subject, sec.signin_account_cooldown_minutes):
+            minutes = max(1, (wait + 59) // 60)
+            _security_audit(
+                conn,
+                "auth.lockout",
+                subject,
+                {"account": account, "kind": kind, "minutes": minutes},
+                user_id=user_id,
+                ip=ip,
+            )
+            notify(
+                conn,
+                SECURITY_ALERT,
+                subject=subject,
+                title=f"Sign-in cooldown started for {account}",
+                body=(
+                    f"{sec.signin_account_limit} failed attempts on {account} within "
+                    f"{sec.signin_account_window_minutes} minutes. Further attempts are refused for {minutes} "
+                    f"minutes. Last address: {ip or 'unknown'}."
+                ),
+                detail={"account": account, "kind": kind, "ip": ip, "minutes": minutes},
+                dedupe_minutes=ALERT_DEDUPE_MINUTES,
+            )
+    if ip:
+        wait = ratelimit.check(conn, s, accounts=[], ip=ip, now=now)
+        subject = f"ip:{ip}"
+        if wait is not None and not _recent_audit(conn, "auth.signin_burst", subject, BURST_DEDUPE_MINUTES):
+            _security_audit(conn, "auth.signin_burst", subject, {"reason": "ip_limit", "ip": ip}, ip=ip)
+            notify(
+                conn,
+                SIGNIN_BURST,
+                subject=subject,
+                title=f"Sign-in limit reached for address {ip}",
+                body=(
+                    f"{sec.signin_ip_limit} failed sign-ins from {ip} within {sec.signin_ip_window_minutes} minutes. "
+                    "The address is refused until the count drops."
+                ),
+                detail={"reason": "ip_limit", "ip": ip},
+                dedupe_minutes=BURST_DEDUPE_MINUTES,
+            )
+    if kind in BURST_KINDS:
+        n = conn.execute(
+            "SELECT count(*), count(DISTINCT account) FROM signin_attempts WHERE NOT ok"
+            " AND reason IS DISTINCT FROM %s AND kind = ANY(%s) AND at > %s",
+            (ratelimit.RATE_LIMITED, list(BURST_KINDS), now - timedelta(minutes=secsettings.BURST_WINDOW_MINUTES)),
+        ).fetchone()
+        failures, accounts = int(n[0]), int(n[1])
+        if failures >= sec.signin_burst_threshold and not _recent_audit(
+            conn, "auth.signin_burst", "burst", BURST_DEDUPE_MINUTES
+        ):
+            detail = {"reason": "burst", "failures": failures, "accounts": accounts}
+            _security_audit(conn, "auth.signin_burst", "burst", detail, ip=ip)
+            notify(
+                conn,
+                SIGNIN_BURST,
+                subject="burst",
+                title=f"{failures} failed sign-ins in {secsettings.BURST_WINDOW_MINUTES} minutes",
+                body=(
+                    f"{failures} failed sign-ins across {accounts} accounts in the last "
+                    f"{secsettings.BURST_WINDOW_MINUTES} minutes (the threshold is {sec.signin_burst_threshold})."
+                ),
+                detail=detail,
+                dedupe_minutes=BURST_DEDUPE_MINUTES,
+            )
+
+
+def _name(conn: Any, user_id: Any) -> str:
+    row = conn.execute("SELECT display_name, sam_account FROM users WHERE id = %s", (user_id,)).fetchone()
+    return f"{row[0]} ({row[1]})" if row else str(user_id)
+
+
+def pin_disabled(conn: Any, user_id: Any, failures: int, where: str) -> None:
+    """A PIN was disabled after too many wrong entries."""
+    name = _name(conn, user_id)
+    notify(
+        conn,
+        SECURITY_ALERT,
+        subject=f"user:{user_id}",
+        title=f"PIN disabled after {failures} wrong entries: {name}",
+        body=f"The PIN of {name} was disabled after {failures} wrong entries ({where}). An Admin can clear it.",
+        detail={"user_id": str(user_id), "failures": failures, "where": where},
+        dedupe_minutes=ALERT_DEDUPE_MINUTES,
+    )
+
+
+def stepup_session_ended(conn: Any, user_id: Any, failures: int, kind: str, ip: str | None) -> None:
+    """A session ended after too many wrong unlock or signature attempts."""
+    name = _name(conn, user_id)
+    what = "unlock" if kind == "unlock" else "signature"
+    notify(
+        conn,
+        SECURITY_ALERT,
+        subject=f"user:{user_id}",
+        title=f"Session ended after {failures} wrong {what} attempts: {name}",
+        body=f"{name} entered a wrong PIN or password {failures} times at {what}. The session was ended. "
+        f"Address: {ip or 'unknown'}.",
+        detail={"user_id": str(user_id), "failures": failures, "kind": kind, "ip": ip},
+        dedupe_minutes=ALERT_DEDUPE_MINUTES,
+    )
+
+
+# ---------------------------------------------------------------- policy denials
+
+_denied_lock = threading.Lock()
+_denied_seen: dict[tuple[str, str, str, str], float] = {}
+
+
+def _denied_due(key: tuple[str, str, str, str]) -> bool:
+    now = time.monotonic()
+    with _denied_lock:
+        last = _denied_seen.get(key)
+        if last is not None and now - last < DENIED_DEDUPE_SECONDS:
+            return False
+        if len(_denied_seen) > 5000:
+            _denied_seen.clear()
+        _denied_seen[key] = now
+        return True
+
+
+def record_denied(settings: Settings, principal: Any, *, method: str, path: str, code: str) -> None:
+    """Audit `policy.denied` for a 403 refused to a signed-in principal. The method, path and code only (never a
+    body or a query string). The same user, path and code is audited once a minute at most."""
+    if code in DENIED_SKIP_CODES:
+        return
+    key = (str(principal.user_id), method, path, code)
+    if not _denied_due(key):
+        return
+    from .. import db
+
+    try:
+        with db.get_pool(settings).connection(timeout=5.0) as conn:
+            conn.autocommit = True
+            audit.append(
+                conn,
+                action="policy.denied",
+                user_id=principal.user_id,
+                session_id=principal.session_id,
+                role_snapshot=principal.role_snapshot,
+                subject_type="route",
+                subject_id=f"{method} {path}"[:300],
+                detail={"method": method, "path": path[:300], "code": code},
+                ip=principal.ip,
+            )
+    except Exception as e:
+        log.warning("could not audit a denial: %s", type(e).__name__)
+
+
+# ---------------------------------------------------------------- the Security tab's list
+
+
+def _row(kind: str, r: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "at": r["at"],
+        "kind": kind,
+        "user_id": str(r["user_id"]) if r.get("user_id") else None,
+        "display_name": r.get("display_name"),
+        "account": r.get("account"),
+        "ip": r.get("ip"),
+        "detail": r.get("detail") or {},
+    }
+
+
+AUDIT_ALERTS = (
+    # (kind, WHERE on audit_events e)
+    ("lockout", "e.action = 'auth.lockout'"),
+    ("signin_burst", "e.action = 'auth.signin_burst'"),
+    ("denied", "e.action = 'policy.denied'"),
+    ("admin_change", "e.action LIKE 'admin.%%'"),
+    ("it_edit", "e.it_reason IS NOT NULL"),
+)
+
+
+AUDIT_ALERT_SQL = """
+SELECT e.at, e.user_id, u.display_name, coalesce(e.detail->>'account', u.sam_account) AS account,
+       host(e.ip) AS ip, e.action, e.detail, e.it_reason, e.subject_type, e.subject_id
+FROM audit_events e LEFT JOIN users u ON u.id = e.user_id WHERE {where} AND e.at > %s
+ORDER BY e.id DESC LIMIT %s
+"""
+
+
+def alerts(conn: Any, hours: int) -> list[dict[str, Any]]:
+    """What happened in the last `hours`, newest first, at most ALERTS_MAX rows."""
+    out: list[dict[str, Any]] = []
+    since = conn.execute("SELECT now() - make_interval(hours => %s::int)", (hours,)).fetchone()[0]
+    with conn.cursor(row_factory=dict_row) as cur:
+        for kind, where in AUDIT_ALERTS:
+            rows = cur.execute(
+                AUDIT_ALERT_SQL.format(where=where),
+                (since, ALERTS_MAX),
+            ).fetchall()
+            for r in rows:
+                detail = dict(r["detail"] or {}) if isinstance(r["detail"], dict) else {"value": r["detail"]}
+                detail.setdefault("action", r["action"])
+                if r["subject_type"]:
+                    detail.setdefault("subject", f"{r['subject_type']}:{r['subject_id']}")
+                if kind == "it_edit":
+                    detail["it_reason"] = r["it_reason"]
+                out.append(_row(kind, {**r, "detail": detail}))
+        rows = cur.execute(
+            "SELECT max(a.at) AS at, a.user_id, u.display_name, u.sam_account AS account, count(*) AS n,"
+            " (array_agg(host(a.ip) ORDER BY a.at DESC))[1] AS ip, array_agg(DISTINCT a.kind) AS kinds"
+            " FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
+            " WHERE NOT a.ok AND a.reason = 'bad_pin' AND a.user_id IS NOT NULL AND a.at > %s"
+            " GROUP BY a.user_id, u.display_name, u.sam_account HAVING count(*) >= %s",
+            (since, PIN_FAILURES_MIN),
+        ).fetchall()
+        for r in rows:
+            out.append(_row("pin_failures", {**r, "detail": {"wrong_pins": int(r["n"]), "where": sorted(r["kinds"])}}))
+        rows = cur.execute(
+            "SELECT a.at, a.user_id, u.display_name, u.sam_account AS account, host(a.ip) AS ip, a.kind, a.reason"
+            " FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
+            " WHERE NOT a.ok AND a.kind IN ('unlock', 'signature') AND a.reason IN ('bad_pin', 'bad_password')"
+            " AND a.at > %s ORDER BY a.id DESC LIMIT %s",
+            (since, ALERTS_MAX),
+        ).fetchall()
+        for r in rows:
+            out.append(_row("stepup_wrong_pin", {**r, "detail": {"where": r["kind"], "reason": r["reason"]}}))
+        rows = cur.execute(
+            "SELECT f.at, f.kind, f.subject, f.detail, f.run_id FROM integrity_findings f WHERE f.at > %s"
+            " ORDER BY f.id DESC LIMIT %s",
+            (since, ALERTS_MAX),
+        ).fetchall()
+        for r in rows:
+            detail = {"finding": r["kind"], "subject": r["subject"], "run_id": r["run_id"], **(r["detail"] or {})}
+            out.append(_row("integrity", {"at": r["at"], "detail": detail}))
+    out.sort(key=lambda x: x["at"], reverse=True)
+    return out[:ALERTS_MAX]

server/packetstation/domain/shop.py

@@ -405,14 +405,15 @@ PEOPLE_SQL = """
 SELECT u.id, u.display_name, u.welder_stamp, max(CASE r.role::text WHEN 'shop_supervisor' THEN 3
        WHEN 'shop_lead' THEN 2 ELSE 1 END) AS level
 FROM users u JOIN user_roles r ON r.user_id = u.id
-WHERE u.active AND r.dept_code = %s AND r.role::text IN ('shop_worker', 'shop_lead', 'shop_supervisor')
+WHERE u.active AND u.profile_active AND r.dept_code = %s
+  AND r.role::text IN ('shop_worker', 'shop_lead', 'shop_supervisor')
 GROUP BY u.id, u.display_name, u.welder_stamp
 ORDER BY lower(u.display_name), u.id
 """
 
 
 def people(conn: Any, dept: str) -> list[dict[str, Any]]:
-    """The active users with a shop role in a department, for the welder picker."""
+    """The active users with a shop role in a department, for the welder picker. An inactive profile is left out."""
     found = packets.rows(conn, PEOPLE_SQL, (dept,))
     limits: dict[str, list[int]] = {}
     if found:

server/packetstation/main.py

@@ -97,6 +97,8 @@ def create_app(
     from .api import (
         account,
         admin,
+        admin_org,
+        admin_security,
         assets,
         auth,
         departments,
@@ -120,6 +122,8 @@ def create_app(
         auth,
         account,
         admin,
+        admin_org,
+        admin_security,
         departments,
         events_api,
         projects,
@@ -138,7 +142,7 @@ def create_app(
     ):
         app.include_router(module.router)
     app.include_router(assets.blob_router)
-    for name in ("feed", "packet_visual"):
+    for name in ("feed", "packet_visual", "notifications"):
         modname = f"{__package__}.api.{name}"
         try:
             extra = importlib.import_module(modname)

server/packetstation/migrations/0010_admin_notify.sql

@@ -0,0 +1,162 @@
+-- 0010_admin_notify: phase 9. Admin profiles, notification routing and delivery, integrity findings.
+-- Runs as ps_migrate; 0001's default privileges give ps_app SELECT, INSERT, UPDATE, DELETE on every table here.
+-- Append-only tables revoke below.
+
+-- ---------------------------------------------------------------- profiles and departments
+
+-- The Admin's "active" on a person's profile. users.active belongs to the AD reconcile (disabled or gone in AD);
+-- profile_active is the app's own switch. An inactive profile drops out of the welder picker, the worker filter
+-- and the station pickers. It does not change what the person may do; AD groups decide that.
+ALTER TABLE users ADD COLUMN profile_active bool NOT NULL DEFAULT true;
+
+-- Who added a department in the app. Seeded departments have neither.
+ALTER TABLE departments ADD COLUMN created_at timestamptz;
+ALTER TABLE departments ADD COLUMN created_by uuid REFERENCES users (id);
+
+-- When the integrity job last rehashed the file behind an asset, and whether it matched.
+ALTER TABLE assets ADD COLUMN hash_checked_at timestamptz;
+ALTER TABLE assets ADD COLUMN hash_ok bool;
+
+-- ---------------------------------------------------------------- notification routing
+
+CREATE TYPE notify_delivery AS ENUM ('immediate', 'digest');
+CREATE TYPE notify_recipient_kind AS ENUM ('group', 'user', 'mailbox', 'dept_supervisor');
+
+-- One row per event type (the catalog in notify/catalog.py). Seeded by the code on first start: enabled,
+-- immediate, recipient PS-Admins. param is N hours or N days for the event types that take one.
+CREATE TABLE notification_rules (
+    event_type  text PRIMARY KEY,
+    enabled     bool NOT NULL DEFAULT true,
+    delivery    notify_delivery NOT NULL DEFAULT 'immediate',
+    digest_time time NOT NULL DEFAULT '07:00',
+    param       int CHECK (param IS NULL OR param > 0),
+    version     int NOT NULL DEFAULT 1,
+    updated_at  timestamptz,
+    updated_by  uuid REFERENCES users (id)
+);
+
+-- group: an AD group by DN (any group under LDAP_BASE_DN), resolved to its members' mail at send time.
+-- user: one person from the directory. mailbox: a plain address. dept_supervisor: the event's department
+-- Supervisor group, worked out at send time.
+CREATE TABLE notification_recipients (
+    id         bigserial PRIMARY KEY,
+    event_type text NOT NULL REFERENCES notification_rules (event_type) ON DELETE CASCADE,
+    kind       notify_recipient_kind NOT NULL,
+    group_dn   text,
+    group_name text,
+    user_id    uuid REFERENCES users (id),
+    address    text,
+    CHECK (kind <> 'group' OR group_dn IS NOT NULL),
+    CHECK (kind <> 'user' OR user_id IS NOT NULL),
+    CHECK (kind <> 'mailbox' OR address IS NOT NULL)
+);
+CREATE INDEX notification_recipients_event ON notification_recipients (event_type);
+
+-- A person left out of one event type's delivery. self = set by the person on Account settings (never allowed
+-- for the security event types); otherwise set by an Admin.
+CREATE TABLE notification_exclusions (
+    event_type text NOT NULL REFERENCES notification_rules (event_type) ON DELETE CASCADE,
+    user_id    uuid NOT NULL REFERENCES users (id),
+    self       bool NOT NULL DEFAULT false,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    created_by uuid REFERENCES users (id),
+    PRIMARY KEY (event_type, user_id)
+);
+
+-- ---------------------------------------------------------------- notification events (0005's table, extended)
+
+-- kind stays as the old key and equals event_type for every row raised through notify.raise_event.
+-- A standing condition (feed failing, backup stale, quota) stays open (resolved_at null) until it clears; raising
+-- it again while open only counts (raised_count, last_raised_at) and sends nothing: that is the dedupe. Clearing
+-- it sets resolved_at and inserts a recovery row (recovery_of) that is sent to the same rule. A one-shot event
+-- (security alert, reject, version deployed) is inserted already resolved.
+ALTER TABLE notification_events ADD COLUMN event_type text;
+ALTER TABLE notification_events ADD COLUMN title text;
+ALTER TABLE notification_events ADD COLUMN body text;
+ALTER TABLE notification_events ADD COLUMN dept_code text REFERENCES departments (code);
+ALTER TABLE notification_events ADD COLUMN raised_count int NOT NULL DEFAULT 1;
+ALTER TABLE notification_events ADD COLUMN last_raised_at timestamptz NOT NULL DEFAULT now();
+ALTER TABLE notification_events ADD COLUMN recovery_of bigint REFERENCES notification_events (id);
+-- Set by notify-send when it has handled the row: sent, queued for the digest, or skipped (rule off).
+ALTER TABLE notification_events ADD COLUMN dispatched_at timestamptz;
+ALTER TABLE notification_events ADD COLUMN dispatch_result text;
+
+-- The phase 4 quota rows become assets_quota events. They were raised before delivery existed, so they are
+-- marked handled and never mailed late.
+UPDATE notification_events SET event_type = kind, title = 'Assets storage warning', dispatched_at = now(),
+    dispatch_result = 'before_phase_9'
+WHERE event_type IS NULL;
+
+CREATE INDEX notification_events_pending ON notification_events (id)
+    WHERE dispatched_at IS NULL AND event_type IS NOT NULL;
+CREATE INDEX notification_events_type_at ON notification_events (event_type, created_at DESC);
+
+-- Events waiting for their rule's daily digest. Recipients are resolved when the digest goes, not here.
+CREATE TABLE notification_outbox (
+    id         bigserial PRIMARY KEY,
+    event_id   bigint NOT NULL REFERENCES notification_events (id),
+    event_type text NOT NULL,
+    queued_at  timestamptz NOT NULL DEFAULT now(),
+    sent_at    timestamptz,
+    sent_id    bigint
+);
+CREATE INDEX notification_outbox_pending ON notification_outbox (event_type, id) WHERE sent_at IS NULL;
+
+-- Every message, append-only: the event type ('digest' for a digest), the events in it, the subject, the
+-- recipients as resolved [{address, user_id, why}], and the result. A test send is marked test.
+CREATE TABLE notifications_sent (
+    id           bigserial PRIMARY KEY,
+    at           timestamptz NOT NULL DEFAULT now(),
+    event_type   text NOT NULL,
+    event_ids    bigint[] NOT NULL DEFAULT '{}',
+    digest       bool NOT NULL DEFAULT false,
+    test         bool NOT NULL DEFAULT false,
+    subject      text NOT NULL,
+    recipients   jsonb NOT NULL DEFAULT '[]',
+    result       text NOT NULL CHECK (result IN ('sent', 'dry_run', 'failed', 'no_recipients')),
+    error        text,
+    requested_by uuid REFERENCES users (id)
+);
+CREATE INDEX notifications_sent_type ON notifications_sent (event_type, id DESC);
+
+CREATE TRIGGER notifications_sent_append_only
+    BEFORE UPDATE OR DELETE ON notifications_sent
+    FOR EACH ROW EXECUTE FUNCTION ps_append_only();
+CREATE TRIGGER notifications_sent_no_truncate
+    BEFORE TRUNCATE ON notifications_sent
+    FOR EACH STATEMENT EXECUTE FUNCTION ps_append_only();
+
+-- ---------------------------------------------------------------- integrity
+
+-- What the integrity job found wrong, per run, append-only. kind: seal (a filed packet's seal does not verify),
+-- asset_hash (the file's sha256 differs), asset_missing (no file for an asset row), blob_orphan (a file no asset
+-- row names; reported, never deleted), audit_chain (the hash chain breaks at subject = the audit id).
+CREATE TABLE integrity_findings (
+    id      bigserial PRIMARY KEY,
+    -- no foreign key: housekeeping prunes job_runs at 180 days and these rows are append-only
+    run_id  bigint,
+    at      timestamptz NOT NULL DEFAULT now(),
+    kind    text NOT NULL CHECK (kind IN ('seal', 'asset_hash', 'asset_missing', 'blob_orphan', 'audit_chain')),
+    subject text NOT NULL,
+    detail  jsonb NOT NULL DEFAULT '{}'
+);
+CREATE INDEX integrity_findings_run ON integrity_findings (run_id, id);
+
+CREATE TRIGGER integrity_findings_append_only
+    BEFORE UPDATE OR DELETE ON integrity_findings
+    FOR EACH ROW EXECUTE FUNCTION ps_append_only();
+CREATE TRIGGER integrity_findings_no_truncate
+    BEFORE TRUNCATE ON integrity_findings
+    FOR EACH STATEMENT EXECUTE FUNCTION ps_append_only();
+
+-- ---------------------------------------------------------------- jobs
+
+-- The phase 9 jobs, so "refresh from AD now" is accepted before the worker has started once.
+INSERT INTO job_schedule (job) VALUES
+    ('ad-reconcile'), ('session-refresh'), ('integrity'), ('backup-check'), ('notify-send'), ('notify-digest')
+ON CONFLICT (job) DO NOTHING;
+
+-- Explicit, after the default privileges: the app may only read and append these two.
+REVOKE UPDATE, DELETE, TRUNCATE ON notifications_sent, integrity_findings FROM ps_app;
+GRANT SELECT, INSERT ON notifications_sent, integrity_findings TO ps_app;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ps_app;

server/packetstation/notify/__init__.py

@@ -0,0 +1,13 @@
+"""Notifications: the event catalog, raising and resolving events, routing, delivery and the digest.
+
+Everyone else calls `raise_event` and `resolve_event` inside their own transaction. Only the notify-send and
+notify-digest jobs (and the Admin's test send) talk to the mail relay.
+"""
+
+from __future__ import annotations
+
+from .catalog import EVENT_TYPES, SECURITY_TYPES, EventType
+from .events import raise_event, resolve_event
+from .seeding import seed
+
+__all__ = ["EVENT_TYPES", "SECURITY_TYPES", "EventType", "raise_event", "resolve_event", "seed"]

server/packetstation/notify/catalog.py

@@ -0,0 +1,82 @@
+"""The notification event types. One routing rule per type lives in `notification_rules`.
+
+A security type cannot be opted out of on Account settings, and the Admin cannot exclude anyone from it either.
+A department-scoped type carries the department of its packet, so it can go to that department's Supervisor group.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class EventType:
+    key: str
+    label: str
+    description: str
+    security: bool = False
+    dept_scoped: bool = False
+    param: str | None = None  # "hours" or "days"
+    param_default: int | None = None
+
+
+_TYPES = (
+    EventType(
+        "security_alert",
+        "Security alert",
+        "An account cooldown started, a PIN was disabled after five wrong tries, or a session ended after three"
+        " wrong unlock or signature PINs.",
+        security=True,
+    ),
+    EventType(
+        "signin_burst",
+        "Sign-in burst",
+        "The per-address sign-in limit tripped, or many sign-ins failed across accounts in five minutes.",
+        security=True,
+    ),
+    EventType(
+        "integrity_failure",
+        "Integrity check failure",
+        "The nightly integrity check found a broken seal, a changed or missing file, or a break in the audit chain.",
+        security=True,
+    ),
+    EventType("feed_failed", "Feed run failed", "A VISUAL feed or drawing lookup run failed."),
+    EventType("feed_degraded", "Feed run degraded", "A VISUAL feed run finished with errors."),
+    EventType(
+        "drawing_failing",
+        "Drawing lookup failing",
+        "Drawing lookups in M-Files have failed for more than the set number of hours.",
+        param="hours",
+        param_default=24,
+    ),
+    EventType(
+        "ad_reconcile_report",
+        "AD reconcile report",
+        "The nightly AD reconcile added or deactivated people, found extra groups, or flagged employee IDs.",
+    ),
+    EventType("group_missing", "Group missing from the OU", "A mapped PS group is not in the PacketStation OU."),
+    EventType("backup_stale", "Backup failed or stale", "The newest database backup is missing, empty, old or failed."),
+    EventType(
+        "co_waiting",
+        "Change order waiting",
+        "A change order has waited for review for more than the set number of days.",
+        dept_scoped=True,
+        param="days",
+        param_default=3,
+    ),
+    EventType(
+        "qc_reject",
+        "Hold-point or ITP reject",
+        "Q.C. rejected a packet, or an ITP line was rejected.",
+        dept_scoped=True,
+    ),
+    EventType("version_deployed", "Version deployed", "The worker started on a new build."),
+    EventType("assets_quota", "Assets storage warning", "The assets storage passed its warning threshold."),
+)
+
+EVENT_TYPES: dict[str, EventType] = {t.key: t for t in _TYPES}
+SECURITY_TYPES: frozenset[str] = frozenset(t.key for t in _TYPES if t.security)
+
+
+def is_known(event_type: str) -> bool:
+    return event_type in EVENT_TYPES

server/packetstation/notify/digest.py

@@ -0,0 +1,154 @@
+"""notify-digest: once a rule's digest time has passed today (site time), send what it queued before then.
+
+Recipients are resolved per event, then grouped per address: one message per address listing every event due for
+it, across rules. One notifications_sent row per message (event_type 'digest'). An outbox row is marked sent once
+every address it goes to has it; a failed send leaves it for the next run, and an address that already has an
+event is not sent it again.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, time, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from psycopg.rows import dict_row
+
+from ..settings import Settings
+from . import smtp
+from .routing import DIRECTORY_DOWN, DirectoryDown, NotifyDirectory, Recipient, Resolver, load_rules
+from .send import deliver, event_lines, footer, record
+
+log = logging.getLogger(__name__)
+
+DEFAULT_TIME = time(7, 0)
+BACKOFF = timedelta(minutes=30)
+
+
+def _due_rows(conn: Any, settings: Settings, rules: dict[str, Any], now: datetime) -> list[dict[str, Any]]:
+    tz = ZoneInfo(settings.site_tz)
+    pending = [r[0] for r in conn.execute("SELECT DISTINCT event_type FROM notification_outbox WHERE sent_at IS NULL")]
+    out: list[dict[str, Any]] = []
+    for et in pending:
+        rule = rules.get(et)
+        at = rule.digest_time if rule is not None else DEFAULT_TIME
+        cutoff = datetime.combine(now.astimezone(tz).date(), at, tzinfo=tz)
+        if now < cutoff:
+            continue
+        with conn.cursor(row_factory=dict_row) as cur:
+            out.extend(
+                dict(r)
+                for r in cur.execute(
+                    "SELECT o.id AS outbox_id, o.queued_at, e.* FROM notification_outbox o"
+                    " JOIN notification_events e ON e.id = o.event_id"
+                    " WHERE o.sent_at IS NULL AND o.event_type = %s AND o.queued_at < %s ORDER BY o.id",
+                    (et, cutoff),
+                ).fetchall()
+            )
+    return out
+
+
+def _already(conn: Any, event_ids: list[int]) -> set[tuple[int, str]]:
+    """(event id, address) pairs a digest already carried, sent or recorded in dry run."""
+    done: set[tuple[int, str]] = set()
+    for ids, recips in conn.execute(
+        "SELECT event_ids, recipients FROM notifications_sent WHERE digest AND result IN ('sent', 'dry_run')"
+        " AND event_ids && %s::bigint[]",
+        (event_ids,),
+    ).fetchall():
+        for rc in recips or []:
+            addr = str(rc.get("address") or "").lower()
+            done.update((int(i), addr) for i in ids or [])
+    return done
+
+
+def run_digest(
+    conn: Any, settings: Settings, directory: NotifyDirectory | None, now: datetime | None = None
+) -> dict[str, Any]:
+    """Send the due digests. Returns counts."""
+    tz = ZoneInfo(settings.site_tz)
+    now = now or datetime.now(tz)
+    last = conn.execute(
+        "SELECT result, at FROM notifications_sent WHERE digest AND NOT test ORDER BY id DESC LIMIT 1"
+    ).fetchone()
+    rules = load_rules(conn)
+    rows = _due_rows(conn, settings, rules, now)
+    if not rows:
+        return {"due": 0}
+    if last is not None and last[0] == "failed" and now - last[1] < BACKOFF:
+        return {"due": len(rows), "backoff": True}
+    resolver = Resolver(conn, settings, directory)
+    per_event: dict[int, list[Recipient]] = {}
+    try:
+        for r in rows:
+            rule = rules.get(str(r["event_type"]))
+            per_event[int(r["id"])] = resolver.resolve(rule, r.get("dept_code")) if rule and rule.enabled else []
+    except DirectoryDown:
+        ids = sorted({int(r["id"]) for r in rows})
+        subject = smtp.full_subject(settings, "Daily digest")
+        record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[],
+            result="failed",
+            error=DIRECTORY_DOWN,
+            event_ids=ids,
+            digest=True,
+        )
+        return {"due": len(rows), "failed": 1, "error": DIRECTORY_DOWN}
+    done = _already(conn, sorted(per_event))
+    by_addr: dict[str, tuple[Recipient, list[dict[str, Any]]]] = {}
+    seen_event: dict[str, set[int]] = {}
+    for r in rows:
+        eid = int(r["id"])
+        for rc in per_event[eid]:
+            key = rc.address.lower()
+            if (eid, key) in done or eid in seen_event.setdefault(key, set()):
+                continue
+            seen_event[key].add(eid)
+            by_addr.setdefault(key, (rc, []))[1].append(r)
+    failed_events: set[int] = set()
+    sent_by_event: dict[int, int] = {}
+    counts: dict[str, Any] = {"due": len(rows), "messages": 0}
+    for _key, (rc, events) in sorted(by_addr.items()):
+        n = len(events)
+        subject = smtp.full_subject(settings, f"Daily digest: {n} notification{'s' if n != 1 else ''}")
+        lines: list[str] = [f"{n} Packet Station notification{'s' if n != 1 else ''} since the last digest.", ""]
+        for i, ev in enumerate(events, 1):
+            block = event_lines(settings, ev)
+            lines.append(f"{i}. {block[0]}")
+            lines.extend(f"   {x}" if x else "" for x in block[1:])
+            lines.append("")
+        lines.extend(footer(settings))
+        ids = [int(e["id"]) for e in events]
+        result, error = deliver(settings, subject, "\n".join(lines), [rc])
+        sid = record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[rc],
+            result=result,
+            error=error,
+            event_ids=ids,
+            digest=True,
+        )
+        counts["messages"] += 1
+        counts[result] = counts.get(result, 0) + 1
+        if result in ("sent", "dry_run"):
+            for i in ids:
+                sent_by_event[i] = sid
+        else:
+            failed_events.update(ids)
+    with conn.transaction():
+        for r in rows:
+            eid = int(r["id"])
+            if eid in failed_events:
+                continue
+            conn.execute(
+                "UPDATE notification_outbox SET sent_at = now(), sent_id = %s WHERE id = %s AND sent_at IS NULL",
+                (sent_by_event.get(eid), r["outbox_id"]),
+            )
+    counts["left"] = len({int(r["id"]) for r in rows if int(r["id"]) in failed_events})
+    return counts

server/packetstation/notify/events.py

@@ -0,0 +1,134 @@
+"""Raise and resolve notification events inside the caller's transaction. No I/O here: notify-send mails them.
+
+A standing condition stays open (resolved_at null) until it clears. Raising it again while open only counts it.
+Clearing it closes the row and adds a recovery row that goes to the same rule. A one-shot is inserted resolved.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .catalog import EVENT_TYPES
+
+log = logging.getLogger(__name__)
+
+SEVERITIES = ("info", "warning", "critical")
+TITLE_MAX = 200
+BODY_MAX = 8000
+SUBJECT_MAX = 200
+
+
+def _jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value if value is not None else {}, default=str))
+
+
+def _clip(text: str | None, n: int) -> str:
+    t = (text or "").strip()
+    return t if len(t) <= n else t[: n - 1] + "…"
+
+
+def raise_event(
+    conn: Any,
+    event_type: str,
+    *,
+    subject: str,
+    title: str,
+    body: str = "",
+    detail: dict[str, Any] | None = None,
+    dept_code: str | None = None,
+    severity: str = "warning",
+    condition: bool = False,
+    dedupe_minutes: int = 0,
+) -> int | None:
+    """Record one event. Returns the new row's id, or None when it was deduped (or could not be recorded).
+
+    Never raises: a notification must not break a sign-in, a reject or a job. A failure is logged.
+    """
+    if event_type not in EVENT_TYPES:
+        log.error("raise_event: unknown event type %r; not recorded", event_type)
+        return None
+    if severity not in SEVERITIES:
+        severity = "warning"
+    subj = _clip(subject, SUBJECT_MAX)
+    args = (_clip(title, TITLE_MAX), _clip(body, BODY_MAX), Jsonb(_jsonable(detail)), dept_code)
+    try:
+        with conn.transaction():
+            if condition:
+                row = conn.execute(
+                    "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                    " dept_code) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
+                    " ON CONFLICT (kind, coalesce(subject, '')) WHERE resolved_at IS NULL DO UPDATE SET"
+                    " raised_count = notification_events.raised_count + 1, last_raised_at = now(),"
+                    " detail = EXCLUDED.detail, body = EXCLUDED.body"
+                    " RETURNING id, (xmax = 0) AS inserted",
+                    (event_type, event_type, severity, subj, *args),
+                ).fetchone()
+                return int(row[0]) if row is not None and row[1] else None
+            if dedupe_minutes > 0:
+                recent = conn.execute(
+                    "SELECT 1 FROM notification_events WHERE event_type = %s AND subject = %s"
+                    " AND created_at > now() - make_interval(mins => %s) LIMIT 1",
+                    (event_type, subj, int(dedupe_minutes)),
+                ).fetchone()
+                if recent is not None:
+                    return None
+            row = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now()) RETURNING id",
+                (event_type, event_type, severity, subj, *args),
+            ).fetchone()
+            return int(row[0])
+    except Exception:
+        log.exception("raise_event %s failed; not recorded", event_type)
+        return None
+
+
+def resolve_event(conn: Any, event_type: str, *, subject: str, title: str | None = None, body: str = "") -> int | None:
+    """Close the open condition for (event_type, subject) and add its recovery row. Returns the recovery row's id,
+    or None when nothing was open. Never raises."""
+    if event_type not in EVENT_TYPES:
+        log.error("resolve_event: unknown event type %r", event_type)
+        return None
+    subj = _clip(subject, SUBJECT_MAX)
+    try:
+        with conn.transaction():
+            row = conn.execute(
+                "UPDATE notification_events SET resolved_at = now() WHERE kind = %s AND coalesce(subject, '') = %s"
+                " AND resolved_at IS NULL RETURNING id, title, dept_code",
+                (event_type, subj),
+            ).fetchone()
+            if row is None:
+                return None
+            open_id, open_title, dept = row
+            base = title or open_title or EVENT_TYPES[event_type].label
+            rec = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at, recovery_of) VALUES (%s, %s, 'info', %s, %s, %s, %s, %s, now(), %s)"
+                " RETURNING id",
+                (
+                    event_type,
+                    event_type,
+                    subj,
+                    _clip(f"Cleared: {base}", TITLE_MAX),
+                    _clip(body, BODY_MAX),
+                    Jsonb({"recovery_of": open_id}),
+                    dept,
+                    open_id,
+                ),
+            ).fetchone()
+            return int(rec[0])
+    except Exception:
+        log.exception("resolve_event %s failed", event_type)
+        return None
+
+
+def open_event(conn: Any, event_type: str, subject: str) -> int | None:
+    row = conn.execute(
+        "SELECT id FROM notification_events WHERE kind = %s AND coalesce(subject, '') = %s AND resolved_at IS NULL",
+        (event_type, subject),
+    ).fetchone()
+    return int(row[0]) if row else None

server/packetstation/notify/routing.py

@@ -0,0 +1,222 @@
+"""Rules and recipients. Recipients are resolved when a message goes, so a group stays current.
+
+group: the members of an AD group by DN (nested, enabled accounts with mail). user: users.email of an active user.
+mailbox: the address. dept_supervisor: the Supervisor group of the event's department, none without one.
+Exclusions drop a person by user id and by that person's email.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from dataclasses import dataclass, field
+from datetime import time
+from typing import Any, Protocol
+
+from psycopg.rows import dict_row
+
+from ..auth import groups as groupmap
+from ..auth.ldap import AdUser
+from ..settings import Settings
+from .catalog import EVENT_TYPES
+
+log = logging.getLogger(__name__)
+
+ADDRESS_RE = re.compile(r"^[^@\s<>,;\"']{1,64}@[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251})\.[A-Za-z]{2,63}$")
+DIRECTORY_DOWN = "directory unavailable"
+
+
+class NotifyDirectory(Protocol):
+    """The directory calls notifications use (a subset of auth.ldap.DirectoryProtocol)."""
+
+    def get_user_by_guid(self, guid: uuid.UUID) -> AdUser | None: ...
+    def groups_of(self, dn: str) -> list[str]: ...
+    def members_of_dn(self, group_dn: str) -> list[AdUser]: ...
+    def search_groups(self, q: str, limit: int = 25) -> list[dict[str, Any]]: ...
+
+
+class DirectoryDown(Exception):
+    """AD could not be read while resolving a group."""
+
+
+def address_ok(value: str) -> bool:
+    v = value.strip()
+    return len(v) <= 254 and ADDRESS_RE.match(v) is not None
+
+
+def cn_of(dn: str) -> str:
+    """The first CN of a DN ("CN=PS-Admins,OU=..." -> "PS-Admins"); the DN itself when it has none."""
+    first = dn.split(",", 1)[0].strip()
+    return first[3:].strip() if first[:3].lower() == "cn=" else dn
+
+
+def dn_under(dn: str, base: str) -> bool:
+    b = base.strip().lower().replace(" ", "")
+    return bool(b) and dn.strip().lower().replace(" ", "").endswith("," + b)
+
+
+def supervisor_group(dept_code: str | None) -> str | None:
+    """The Supervisor group name of a department, from the group map (None when it has none)."""
+    if not dept_code:
+        return None
+    for name, role in groupmap.current().entries:
+        if role.dept_code == dept_code and role.kind.endswith("supervisor"):
+            return name
+    return None
+
+
+def group_dn(settings: Settings, name: str) -> str:
+    base = settings.ldap_group_base_dn.strip()
+    return f"CN={name},{base}" if base else f"CN={name}"
+
+
+def supervisor_depts(roles: list[str]) -> list[str]:
+    """The departments a role set supervises (their Supervisor group), sorted."""
+    out: set[str] = set()
+    for text in roles:
+        try:
+            r = groupmap.parse_role(text)
+        except ValueError:
+            continue
+        if r.dept_code and r.kind.endswith("supervisor"):
+            out.add(r.dept_code)
+    return sorted(out)
+
+
+# ---------------------------------------------------------------- rules
+
+
+@dataclass
+class Rule:
+    event_type: str
+    enabled: bool
+    delivery: str
+    digest_time: time
+    param: int | None
+    version: int
+    recipients: list[dict[str, Any]] = field(default_factory=list)
+    exclusions: list[dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def param_value(self) -> int | None:
+        et = EVENT_TYPES.get(self.event_type)
+        if et is None or et.param is None:
+            return None
+        return self.param if self.param is not None else et.param_default
+
+
+def load_rules(conn: Any, event_type: str | None = None) -> dict[str, Rule]:
+    args = (event_type, event_type)
+    with conn.cursor(row_factory=dict_row) as cur:
+        rules = {
+            r["event_type"]: Rule(
+                r["event_type"],
+                bool(r["enabled"]),
+                str(r["delivery"]),
+                r["digest_time"],
+                r["param"],
+                int(r["version"]),
+            )
+            for r in cur.execute(
+                "SELECT event_type, enabled, delivery::text AS delivery, digest_time, param, version"
+                " FROM notification_rules WHERE %s::text IS NULL OR event_type = %s",
+                args,
+            ).fetchall()
+        }
+        for r in cur.execute(
+            "SELECT r.id, r.event_type, r.kind::text AS kind, r.group_dn, r.group_name, r.user_id, r.address,"
+            " u.display_name FROM notification_recipients r LEFT JOIN users u ON u.id = r.user_id"
+            " WHERE %s::text IS NULL OR r.event_type = %s ORDER BY r.id",
+            args,
+        ).fetchall():
+            if r["event_type"] in rules:
+                rules[r["event_type"]].recipients.append(dict(r))
+        for r in cur.execute(
+            "SELECT x.event_type, x.user_id, x.self, u.display_name, u.email FROM notification_exclusions x"
+            " LEFT JOIN users u ON u.id = x.user_id WHERE %s::text IS NULL OR x.event_type = %s"
+            " ORDER BY lower(u.display_name), x.user_id",
+            args,
+        ).fetchall():
+            if r["event_type"] in rules:
+                rules[r["event_type"]].exclusions.append(dict(r))
+    return rules
+
+
+def load_rule(conn: Any, event_type: str) -> Rule | None:
+    return load_rules(conn, event_type).get(event_type)
+
+
+# ---------------------------------------------------------------- resolving
+
+
+@dataclass
+class Recipient:
+    address: str
+    user_id: str | None
+    why: str
+
+    def as_json(self) -> dict[str, Any]:
+        return {"address": self.address, "user_id": self.user_id, "why": self.why}
+
+
+class Resolver:
+    """Resolves rules to addresses. Caches group members for its lifetime (one run or one request)."""
+
+    def __init__(self, conn: Any, settings: Settings, directory: NotifyDirectory | None):
+        self.conn = conn
+        self.settings = settings
+        self.directory = directory
+        self._members: dict[str, list[AdUser]] = {}
+
+    def members(self, dn: str) -> list[AdUser]:
+        key = dn.strip().lower()
+        if key not in self._members:
+            if self.directory is None:
+                raise DirectoryDown(DIRECTORY_DOWN)
+            try:
+                self._members[key] = list(self.directory.members_of_dn(dn))
+            except Exception as e:
+                log.warning("notify: can't read group %s: %s", cn_of(dn), type(e).__name__)
+                raise DirectoryDown(DIRECTORY_DOWN) from None
+        return self._members[key]
+
+    def _group(self, dn: str, why: str) -> list[Recipient]:
+        return [
+            Recipient(u.email.strip(), str(u.guid), why)
+            for u in self.members(dn)
+            if not u.disabled and u.email and u.email.strip()
+        ]
+
+    def resolve(self, rule: Rule, dept_code: str | None) -> list[Recipient]:
+        """Every address the rule reaches for an event of this department, after exclusions, one per address.
+
+        Raises DirectoryDown when a group cannot be read."""
+        found: list[Recipient] = []
+        for r in rule.recipients:
+            kind = r["kind"]
+            if kind == "group" and r.get("group_dn"):
+                found.extend(self._group(r["group_dn"], f"group {r.get('group_name') or cn_of(r['group_dn'])}"))
+            elif kind == "user" and r.get("user_id"):
+                row = self.conn.execute(
+                    "SELECT email FROM users WHERE id = %s AND active AND coalesce(email, '') <> ''", (r["user_id"],)
+                ).fetchone()
+                if row is not None:
+                    found.append(Recipient(str(row[0]).strip(), str(r["user_id"]), "user"))
+            elif kind == "mailbox" and r.get("address"):
+                found.append(Recipient(str(r["address"]).strip(), None, "mailbox"))
+            elif kind == "dept_supervisor":
+                name = supervisor_group(dept_code)
+                if name:
+                    found.extend(self._group(group_dn(self.settings, name), f"Supervisor group of {dept_code}"))
+        excluded_ids = {str(x["user_id"]) for x in rule.exclusions}
+        excluded_mail = {str(x["email"]).strip().lower() for x in rule.exclusions if x.get("email")}
+        out: list[Recipient] = []
+        seen: set[str] = set()
+        for rc in found:
+            key = rc.address.lower()
+            if key in seen or (rc.user_id and rc.user_id in excluded_ids) or key in excluded_mail:
+                continue
+            seen.add(key)
+            out.append(rc)
+        return out

server/packetstation/notify/seeding.py

@@ -0,0 +1,38 @@
+"""Seed the routing rules: every missing event type enabled, immediate, digest 07:00, to PS-Admins.
+
+Idempotent. An existing rule is never touched, so the Admin's choices stand.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..auth.groups import ADMIN_GROUP
+from ..settings import Settings
+from .catalog import EVENT_TYPES
+
+
+def admins_dn(settings: Settings) -> str:
+    base = settings.ldap_group_base_dn.strip()
+    return f"CN={ADMIN_GROUP},{base}" if base else f"CN={ADMIN_GROUP}"
+
+
+def seed(conn: Any, settings: Settings) -> list[str]:
+    """Insert the missing rules. Returns the event types it inserted."""
+    added: list[str] = []
+    with conn.transaction():
+        for key, et in EVENT_TYPES.items():
+            row = conn.execute(
+                "INSERT INTO notification_rules (event_type, enabled, delivery, digest_time, param)"
+                " VALUES (%s, true, 'immediate', '07:00', %s) ON CONFLICT (event_type) DO NOTHING RETURNING event_type",
+                (key, et.param_default),
+            ).fetchone()
+            if row is None:
+                continue
+            conn.execute(
+                "INSERT INTO notification_recipients (event_type, kind, group_dn, group_name)"
+                " VALUES (%s, 'group', %s, %s)",
+                (key, admins_dn(settings), ADMIN_GROUP),
+            )
+            added.append(key)
+    return added

server/packetstation/notify/send.py

@@ -0,0 +1,264 @@
+"""notify-send: deliver the events nobody has handled yet, and the Admin's test send.
+
+Per event: rule off -> dispatched `rule_off`; immediate -> one message to the resolved recipients, one
+notifications_sent row, dispatched with the result; digest -> an outbox row, dispatched `queued`.
+NOTIFY_DRY_RUN records `dry_run` and sends nothing, except a test send, which always goes.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from ..settings import Settings
+from . import smtp
+from .catalog import EVENT_TYPES
+from .routing import DIRECTORY_DOWN, DirectoryDown, NotifyDirectory, Recipient, Resolver, load_rules
+
+log = logging.getLogger(__name__)
+
+MAX_DIRECTORY_TRIES = 3
+RETRY_PREFIX = "directory_retry:"
+BATCH = 200
+FOOTER = "Sent by Packet Station. Who gets this is set on the Admin page, Notifications tab."
+
+
+def record(
+    conn: Any,
+    *,
+    event_type: str,
+    subject: str,
+    recipients: Sequence[Recipient],
+    result: str,
+    error: str | None = None,
+    event_ids: Sequence[int] = (),
+    digest: bool = False,
+    test: bool = False,
+    requested_by: Any = None,
+) -> int:
+    row = conn.execute(
+        "INSERT INTO notifications_sent (event_type, event_ids, digest, test, subject, recipients, result, error,"
+        " requested_by) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id",
+        (
+            event_type,
+            list(event_ids),
+            digest,
+            test,
+            subject,
+            Jsonb([r.as_json() for r in recipients]),
+            result,
+            (error or None) and str(error)[:1000],
+            requested_by,
+        ),
+    ).fetchone()
+    return int(row[0])
+
+
+def deliver(
+    settings: Settings, subject: str, body: str, recipients: Sequence[Recipient], *, test: bool = False
+) -> tuple[str, str | None]:
+    """(result, error) for one message. No recipients -> no_recipients; dry run (not a test) -> dry_run."""
+    if not recipients:
+        return "no_recipients", None
+    if settings.notify_dry_run and not test:
+        return "dry_run", None
+    try:
+        note = smtp.send(settings, subject, body, [r.address for r in recipients])
+    except smtp.MailError as e:
+        log.warning("notify: mail not sent: %s", e)
+        return "failed", str(e)
+    return "sent", note
+
+
+def site_time(settings: Settings, at: datetime | None) -> str:
+    if at is None:
+        return ""
+    return at.astimezone(ZoneInfo(settings.site_tz)).strftime("%Y-%m-%d %H:%M")
+
+
+def label_of(event_type: str) -> str:
+    et = EVENT_TYPES.get(event_type)
+    return et.label if et else event_type
+
+
+def event_title(ev: dict[str, Any]) -> str:
+    return str(ev.get("title") or label_of(str(ev.get("event_type"))))
+
+
+def event_lines(settings: Settings, ev: dict[str, Any]) -> list[str]:
+    """The plain-text block for one event (a whole immediate message, or one entry in a digest)."""
+    lines = [event_title(ev)]
+    if ev.get("body"):
+        lines += ["", str(ev["body"]).rstrip()]
+    lines.append("")
+    lines.append(f"Event: {label_of(str(ev.get('event_type')))}")
+    if ev.get("subject"):
+        lines.append(f"About: {ev['subject']}")
+    if ev.get("dept_code"):
+        lines.append(f"Department: {ev['dept_code']}")
+    lines.append(f"When: {site_time(settings, ev.get('created_at'))} ({settings.site_tz})")
+    if int(ev.get("raised_count") or 1) > 1:
+        lines.append(f"Seen {ev['raised_count']} times, last {site_time(settings, ev.get('last_raised_at'))}")
+    return lines
+
+
+def footer(settings: Settings) -> list[str]:
+    return ["", "--", f"{FOOTER} Stack {settings.stack}, build {settings.ps_build}."]
+
+
+def immediate_text(settings: Settings, ev: dict[str, Any]) -> tuple[str, str]:
+    return smtp.full_subject(settings, event_title(ev)), "\n".join(event_lines(settings, ev) + footer(settings))
+
+
+def _tries(result: Any) -> int:
+    text = str(result or "")
+    if text.startswith(RETRY_PREFIX):
+        try:
+            return int(text[len(RETRY_PREFIX) :])
+        except ValueError:
+            return 0
+    return 0
+
+
+def dispatch_pending(conn: Any, settings: Settings, directory: NotifyDirectory | None) -> dict[str, int]:
+    """Handle every undispatched event once. Returns counts by outcome."""
+    counts: dict[str, int] = {}
+    ids = [
+        int(r[0])
+        for r in conn.execute(
+            "SELECT id FROM notification_events WHERE dispatched_at IS NULL AND event_type IS NOT NULL"
+            " ORDER BY id LIMIT %s",
+            (BATCH,),
+        ).fetchall()
+    ]
+    if not ids:
+        return counts
+    rules = load_rules(conn)
+    resolver = Resolver(conn, settings, directory)
+    for eid in ids:
+        with conn.transaction(), conn.cursor(row_factory=dict_row) as cur:
+            ev = cur.execute(
+                "SELECT * FROM notification_events WHERE id = %s AND dispatched_at IS NULL FOR UPDATE SKIP LOCKED",
+                (eid,),
+            ).fetchone()
+            if ev is None:
+                continue
+            outcome = _dispatch_one(conn, settings, resolver, rules.get(str(ev["event_type"])), dict(ev))
+        counts[outcome] = counts.get(outcome, 0) + 1
+    return counts
+
+
+def _done(conn: Any, eid: int, result: str) -> None:
+    conn.execute(
+        "UPDATE notification_events SET dispatched_at = now(), dispatch_result = %s WHERE id = %s", (result, eid)
+    )
+
+
+def _dispatch_one(conn: Any, settings: Settings, resolver: Resolver, rule: Any, ev: dict[str, Any]) -> str:
+    eid = int(ev["id"])
+    if rule is None:
+        _done(conn, eid, "no_rule")
+        return "no_rule"
+    if not rule.enabled:
+        _done(conn, eid, "rule_off")
+        return "rule_off"
+    if rule.delivery == "digest":
+        conn.execute("INSERT INTO notification_outbox (event_id, event_type) VALUES (%s, %s)", (eid, ev["event_type"]))
+        _done(conn, eid, "queued")
+        return "queued"
+    subject, body = immediate_text(settings, ev)
+    try:
+        recipients = resolver.resolve(rule, ev.get("dept_code"))
+    except DirectoryDown:
+        tries = _tries(ev.get("dispatch_result")) + 1
+        record(
+            conn,
+            event_type=ev["event_type"],
+            subject=subject,
+            recipients=[],
+            result="failed",
+            error=DIRECTORY_DOWN,
+            event_ids=[eid],
+        )
+        if tries >= MAX_DIRECTORY_TRIES:
+            _done(conn, eid, "failed")
+            return "failed"
+        conn.execute(
+            "UPDATE notification_events SET dispatch_result = %s WHERE id = %s", (f"{RETRY_PREFIX}{tries}", eid)
+        )
+        return "retry"
+    result, error = deliver(settings, subject, body, recipients)
+    record(
+        conn,
+        event_type=ev["event_type"],
+        subject=subject,
+        recipients=recipients,
+        result=result,
+        error=error,
+        event_ids=[eid],
+    )
+    _done(conn, eid, result)
+    return result
+
+
+# ---------------------------------------------------------------- test send
+
+
+def send_test(
+    conn: Any,
+    settings: Settings,
+    directory: NotifyDirectory | None,
+    event_type: str,
+    *,
+    to: str | None = None,
+    requested_by: Any = None,
+) -> dict[str, Any]:
+    """Send a test of one event type now, even in dry run. With `to`, only to that address."""
+    et = EVENT_TYPES[event_type]
+    subject = smtp.full_subject(settings, f"Test: {et.label}")
+    body = "\n".join(
+        [
+            f'This is a test of the Packet Station notification "{et.label}".',
+            "",
+            et.description,
+            "",
+            "Nothing has happened. An Admin sent this from the Notifications tab.",
+            *footer(settings),
+        ]
+    )
+    recipients: list[Recipient] = []
+    error: str | None = None
+    result: str | None = None
+    if to:
+        recipients = [Recipient(to.strip(), None, "test address")]
+    else:
+        rule = load_rules(conn, event_type).get(event_type)
+        try:
+            recipients = Resolver(conn, settings, directory).resolve(rule, None) if rule else []
+        except DirectoryDown:
+            result, error = "failed", DIRECTORY_DOWN
+    if result is None:
+        result, error = deliver(settings, subject, body, recipients, test=True)
+    sent_id = record(
+        conn,
+        event_type=event_type,
+        subject=subject,
+        recipients=recipients,
+        result=result,
+        error=error,
+        test=True,
+        requested_by=requested_by,
+    )
+    return {
+        "result": result,
+        "subject": subject,
+        "recipients": [r.as_json() for r in recipients],
+        "error": error,
+        "sent_id": sent_id,
+    }

server/packetstation/notify/smtp.py

@@ -0,0 +1,86 @@
+"""Send one message through the relay. STARTTLS verified against SMTP_CA_BUNDLE, never a plain-text fallback.
+
+To: is the From address; the real recipients go in the envelope only, so nobody sees the list. No credentials.
+"""
+
+from __future__ import annotations
+
+import smtplib
+import ssl
+from collections.abc import Sequence
+from email.message import EmailMessage
+from email.utils import formatdate, make_msgid
+
+from ..settings import Settings
+
+TIMEOUT_SECONDS = 15
+MAIL_OFF = "Mail is off: SMTP_HOST is empty."
+
+
+class MailError(Exception):
+    """The relay could not take the message. The text is safe to store and show (no secrets)."""
+
+
+def tls_context(settings: Settings) -> ssl.SSLContext:
+    """The same verified context the doctor uses: system trust plus SMTP_CA_BUNDLE when set."""
+    return ssl.create_default_context(cafile=str(settings.smtp_ca_bundle) if settings.smtp_ca_bundle else None)
+
+
+def subject_prefix(settings: Settings) -> str:
+    return "[Packet Station]" if settings.stack == "prod" else "[Packet Station TEST]"
+
+
+def full_subject(settings: Settings, title: str) -> str:
+    return f"{subject_prefix(settings)} {title}".strip()[:250]
+
+
+def build_message(settings: Settings, subject: str, body: str) -> EmailMessage:
+    msg = EmailMessage()
+    msg["From"] = settings.smtp_from
+    msg["To"] = settings.smtp_from
+    msg["Subject"] = subject
+    msg["Date"] = formatdate(localtime=False)
+    domain = settings.smtp_from.rpartition("@")[2] or None
+    msg["Message-ID"] = make_msgid(domain=domain)
+    msg["Auto-Submitted"] = "auto-generated"
+    msg.set_content(body or subject)
+    return msg
+
+
+def send(settings: Settings, subject: str, body: str, to: Sequence[str]) -> str | None:
+    """Deliver one message to the envelope addresses `to`. Raises MailError when nothing was delivered.
+
+    Returns None, or a note naming the addresses the relay refused when it took the message for the rest."""
+    host = (settings.smtp_host or "").strip()
+    if not host:
+        raise MailError(MAIL_OFF)
+    if not to:
+        raise MailError("No recipients.")
+    msg = build_message(settings, subject, body)
+    try:
+        with smtplib.SMTP(host, settings.smtp_port, timeout=TIMEOUT_SECONDS) as smtp:
+            smtp.ehlo()
+            if settings.smtp_starttls:
+                # starttls raises SMTPNotSupportedError when the relay does not offer it: no fallback
+                smtp.starttls(context=tls_context(settings))
+                smtp.ehlo()
+            refused = smtp.send_message(msg, from_addr=settings.smtp_from, to_addrs=list(to))
+    except ssl.SSLError as e:
+        raise MailError(f"TLS: {e.reason or e}") from None
+    except ssl.CertificateError as e:
+        raise MailError(f"TLS: {e}") from None
+    except smtplib.SMTPNotSupportedError:
+        raise MailError("The relay does not offer STARTTLS; not sent.") from None
+    except smtplib.SMTPRecipientsRefused as e:
+        raise MailError(f"Every recipient was refused ({len(e.recipients)}).") from None
+    except smtplib.SMTPResponseException as e:
+        raise MailError(f"SMTP {e.smtp_code}: {_text(e.smtp_error)}") from None
+    except (OSError, smtplib.SMTPException) as e:
+        raise MailError(f"{type(e).__name__}: {e}") from None
+    if refused:
+        return f"Refused for {len(refused)} of {len(to)} recipients: {', '.join(sorted(refused))[:300]}"
+    return None
+
+
+def _text(value: bytes | str) -> str:
+    return (value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value))[:300]

server/packetstation/notify/watch.py

@@ -0,0 +1,165 @@
+"""The standing conditions notify-send checks itself (at most every 15 minutes), and the version check.
+
+drawing_failing: drawing lookups failing for more than the rule's hours (subject `drawings`, the list in the body).
+co_waiting: a change order pending review for more than the rule's days (subject `co:<id>`, the packet's department).
+version_deployed: the worker runs a build other than the last one noted in the settings row `notify.last_build`.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .catalog import EVENT_TYPES
+from .events import raise_event, resolve_event
+from .routing import load_rules
+
+log = logging.getLogger(__name__)
+
+BUILD_KEY = "notify.last_build"
+LIST_MAX = 50
+
+
+def _param(conn: Any, event_type: str) -> int:
+    rule = load_rules(conn, event_type).get(event_type)
+    value = rule.param_value if rule is not None else None
+    default = EVENT_TYPES[event_type].param_default or 1
+    return int(value or default)
+
+
+def check_drawings(conn: Any) -> dict[str, Any]:
+    hours = _param(conn, "drawing_failing")
+    rows = conn.execute(
+        "SELECT dwg, failing_since, fail_count, note FROM drawing_lookups"
+        " WHERE failing_since IS NOT NULL AND failing_since < now() - make_interval(hours => %s)"
+        " ORDER BY failing_since, dwg",
+        (hours,),
+    ).fetchall()
+    if not rows:
+        rid = resolve_event(conn, "drawing_failing", subject="drawings", body="Every drawing lookup works again.")
+        return {"drawings_failing": 0, "drawings_cleared": rid is not None}
+    n = len(rows)
+    lines = [f"{r[0]}: failing since {r[1]:%Y-%m-%d %H:%M} UTC, {r[2]} tries. {r[3] or ''}".rstrip() for r in rows]
+    if n > LIST_MAX:
+        lines = [*lines[:LIST_MAX], f"and {n - LIST_MAX} more"]
+    raise_event(
+        conn,
+        "drawing_failing",
+        subject="drawings",
+        title=f"{n} drawing lookup{'s' if n != 1 else ''} failing for more than {hours} hours",
+        body="\n".join(lines),
+        detail={"count": n, "hours": hours, "drawings": [r[0] for r in rows[:LIST_MAX]]},
+        condition=True,
+    )
+    return {"drawings_failing": n}
+
+
+def check_change_orders(conn: Any) -> dict[str, Any]:
+    days = _param(conn, "co_waiting")
+    rows = conn.execute(
+        "SELECT co.id, co.co_no, co.issued_at, p.id, p.dept_code, p.title, p.drawing, pr.number, u.lot,"
+        " floor(extract(epoch FROM now() - co.issued_at) / 86400)::int"
+        " FROM change_orders co JOIN packets p ON p.id = co.packet_id JOIN units u ON u.id = p.unit_id"
+        " JOIN projects pr ON pr.id = u.project_id"
+        " WHERE co.review = 'pending' AND p.removed_at IS NULL AND co.issued_at < now() - make_interval(days => %s)"
+        " ORDER BY co.id",
+        (days,),
+    ).fetchall()
+    waiting = {f"co:{r[0]}" for r in rows}
+    raised = 0
+    for co_id, co_no, _issued, pid, dept, title, drawing, number, lot, age in rows:
+        what = " ".join(x for x in (f"{number}-{lot}", dept, title or drawing or "") if x)
+        new = raise_event(
+            conn,
+            "co_waiting",
+            subject=f"co:{co_id}",
+            title=f"Change order CO-{co_no} waiting {age} days: {what}",
+            body=f"CO-{co_no} on packet {pid} ({what}) has waited for review for {age} days.",
+            detail={"co_id": co_id, "packet_id": pid, "days": age},
+            dept_code=dept,
+            condition=True,
+        )
+        raised += new is not None
+    cleared = 0
+    for (subject,) in conn.execute(
+        "SELECT subject FROM notification_events WHERE kind = 'co_waiting' AND resolved_at IS NULL"
+    ).fetchall():
+        if subject not in waiting:
+            cleared += (
+                resolve_event(conn, "co_waiting", subject=subject, body="Reviewed or no longer waiting.") is not None
+            )
+    return {"co_waiting": len(rows), "co_raised": raised, "co_cleared": cleared}
+
+
+def note_build(conn: Any, build: str) -> int | None:
+    """Raise version_deployed when `build` differs from the last build noted. Returns the event id or None."""
+    build = (build or "").strip()
+    if not build:
+        return None
+    with conn.transaction():
+        row = conn.execute("SELECT value FROM settings WHERE key = %s FOR UPDATE", (BUILD_KEY,)).fetchone()
+        before = (row[0] or {}).get("build") if row is not None else None
+        if before == build:
+            return None
+        if row is None:
+            conn.execute(
+                "INSERT INTO settings (key, value, version, updated_at) VALUES (%s, %s, 1, now())"
+                " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
+                " updated_at = now()",
+                (BUILD_KEY, Jsonb({"build": build})),
+            )
+        else:
+            conn.execute(
+                "UPDATE settings SET value = %s, version = version + 1, updated_at = now() WHERE key = %s",
+                (Jsonb({"build": build}), BUILD_KEY),
+            )
+        log.info("version deployed: %s (was %s)", build, before or "none")
+        return raise_event(
+            conn,
+            "version_deployed",
+            subject=f"build:{build}",
+            title=f"Version {build} deployed",
+            body=f"The worker started on build {build}. The build before was {before or 'not recorded'}.",
+            detail={"build": build, "before": before},
+            severity="info",
+        )
+
+
+FEED_JOBS = ("visual-feed", "drawing-lookup")
+
+
+def feed_run(conn: Any, job: str, status: str, errors: list[str], run_id: int | None = None) -> None:
+    """feed_failed and feed_degraded after a visual-feed or drawing-lookup run."""
+    if job not in FEED_JOBS or status not in ("ok", "degraded", "failed"):
+        return
+    first = errors[0] if errors else ""
+    text = "\n".join(errors[:10])
+    if status == "failed":
+        raise_event(
+            conn,
+            "feed_failed",
+            subject=job,
+            title=f"{job} run failed",
+            body=f"Run {run_id} failed.\n{text}".strip(),
+            detail={"job": job, "run_id": run_id, "error": first},
+            severity="critical",
+            condition=True,
+        )
+        return
+    resolve_event(conn, "feed_failed", subject=job, body=f"Run {run_id} finished {status}.")
+    if job != "visual-feed":
+        return
+    if status == "degraded":
+        raise_event(
+            conn,
+            "feed_degraded",
+            subject=job,
+            title=f"{job} run degraded",
+            body=f"Run {run_id} finished with errors.\n{text}".strip(),
+            detail={"job": job, "run_id": run_id, "error": first},
+            condition=True,
+        )
+    else:
+        resolve_event(conn, "feed_degraded", subject=job, body=f"Run {run_id} finished ok.")

server/packetstation/worker/jobs/__init__.py

@@ -1,7 +1,7 @@
 """The job registry and the shapes every job uses.
 
-Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. The feed and
-drawing jobs are imported lazily by `load_jobs`, so the worker runs before they exist.
+Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. Every job but
+housekeeping is imported lazily by `load_jobs`, so a job module that cannot import is logged and left out.
 """
 
 from __future__ import annotations
@@ -97,7 +97,16 @@ class JobSpec:
 JOBS: dict[str, JobSpec] = {}
 _loaded = False
 _load_lock = threading.Lock()
-LAZY = ("visual_feed", "drawing_lookup")
+LAZY = (
+    "visual_feed",
+    "drawing_lookup",
+    "ad_reconcile",
+    "session_refresh",
+    "integrity",
+    "backup_check",
+    "notify_send",
+    "notify_digest",
+)
 
 
 def register(spec: JobSpec) -> JobSpec:
@@ -117,12 +126,53 @@ def load_jobs() -> dict[str, JobSpec]:
                 try:
                     m = importlib.import_module(f"{__name__}.{mod}")
                     register(m.SPEC)
-                except ImportError as e:
+                except (ImportError, AttributeError) as e:
                     log.warning("job module %s not loaded: %s", mod, redact(str(e)))
             _loaded = True
     return JOBS
 
 
+NOTIFY = __name__.rsplit(".", 2)[0] + ".notify"
+
+
+def directory(ctx: JobContext) -> Any:
+    """The worker's directory (one per process, in the stack cache). Tests put a fake under "directory"."""
+    from ...auth.ldap import Directory
+
+    return ctx.stack.cached("directory", lambda: Directory(ctx.settings))
+
+
+def reload_departments(conn: Any) -> None:
+    """Load the departments table into the group map, so a department the Admin added counts at once."""
+    from ...auth import groups
+
+    deps = groups.load_departments(conn)
+    if deps:
+        groups.use_departments(deps)
+
+
+def raise_event(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.raise_event, or a logged no-op when the notify package is not there."""
+    try:
+        real = importlib.import_module(NOTIFY).raise_event
+    except (ImportError, AttributeError) as e:
+        log.warning("event %s not raised, notify not available: %s", event_type, e)
+        return None
+    result: int | None = real(conn, event_type, **kw)
+    return result
+
+
+def resolve_event(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.resolve_event, or a logged no-op when the notify package is not there."""
+    try:
+        real = importlib.import_module(NOTIFY).resolve_event
+    except (ImportError, AttributeError) as e:
+        log.warning("event %s not resolved, notify not available: %s", event_type, e)
+        return None
+    result: int | None = real(conn, event_type, **kw)
+    return result
+
+
 def seconds_until(now_site: datetime, at: time) -> int:
     """Seconds from now_site to the next `at` (site wall clock), at least 1."""
     target = datetime.combine(now_site.date(), at, tzinfo=now_site.tzinfo)
@@ -140,7 +190,11 @@ __all__ = [
     "JobContext",
     "JobSpec",
     "Stack",
+    "directory",
     "load_jobs",
+    "raise_event",
     "register",
+    "reload_departments",
+    "resolve_event",
     "seconds_until",
 ]

server/packetstation/worker/jobs/ad_reconcile.py

@@ -0,0 +1,133 @@
+"""The AD reconcile, nightly at 01:30 site time and on request ("Refresh from AD now" on the Admin screen).
+
+It reloads the departments, then runs `auth.directory.refresh_from_ad`: list the PacketStation OU, enumerate the
+members of every mapped group (IN_CHAIN, paged at 500), upsert users on objectGUID, mark disabled or gone users
+inactive and end their sessions, end the sessions of users left with no role, and flag missing or duplicate
+employee IDs. The run's counts are the report the Admin screen shows.
+
+Then the notifications: one `group_missing` condition per mapped group not in the OU (resolved once the group is
+there), and one `ad_reconcile_report` when the run created or deactivated anyone, found extra groups in the OU or
+flagged employee IDs. AD unreachable fails the run and changes nothing.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, time
+from typing import Any
+
+from ...auth import directory as userdir
+from ...auth import groups as groupmap
+from ...feed.config import FeedSettings
+from .. import jobs
+from ..redaction import redact
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(1, 30)
+NAME = "ad-reconcile"
+FAILED_SUBJECT = "ad-reconcile:failed"
+BODY_NAMES = 50  # names listed per section in the report mail
+
+
+def _names(items: list[str]) -> str:
+    shown = ", ".join(items[:BODY_NAMES])
+    more = len(items) - BODY_NAMES
+    return shown + (f" and {more} more" if more > 0 else "")
+
+
+def report_text(res: userdir.RefreshResult) -> tuple[str, str] | None:
+    """(title, body) of the reconcile report, or None when there is nothing to report."""
+    parts: list[str] = []
+    lines: list[str] = []
+    if res.created:
+        parts.append(f"{res.created} added")
+        lines.append(f"Added: {_names(res.created_names)}.")
+    if res.deactivated:
+        parts.append(f"{res.deactivated} deactivated")
+        lines.append(f"Deactivated (disabled or gone in AD): {_names(res.deactivated_names)}.")
+    if res.groups_extra:
+        parts.append(f"{len(res.groups_extra)} extra group{'s' if len(res.groups_extra) != 1 else ''}")
+        lines.append(f"Groups in the OU that Packet Station does not map: {_names(res.groups_extra)}.")
+    if res.emp_missing:
+        parts.append(f"{len(res.emp_missing)} without an employee ID")
+        lines.append(f"Accounts without an employee ID: {_names(res.emp_missing)}.")
+    if res.emp_duplicate:
+        parts.append(f"{len(res.emp_duplicate)} duplicate employee ID{'s' if len(res.emp_duplicate) != 1 else ''}")
+        lines.append(f"Employee IDs on more than one account: {_names(res.emp_duplicate)}.")
+    if not parts:
+        return None
+    if res.sessions_ended:
+        lines.append(f"Sessions ended: {res.sessions_ended}.")
+    lines.append(f"Accounts seen in the mapped groups: {res.users_seen}.")
+    return "AD reconcile: " + ", ".join(parts), "\n".join(lines)
+
+
+def notify(conn: Any, res: userdir.RefreshResult) -> dict[str, int]:
+    """Raise and resolve the reconcile's events in one transaction. Returns what was raised."""
+    raised = {"group_missing": 0, "report": 0}
+    missing = {n.lower() for n in res.groups_missing}
+    with conn.transaction():
+        for name in res.groups_missing:
+            jobs.raise_event(
+                conn,
+                "group_missing",
+                subject=name,
+                title=f"Group missing from the OU: {name}",
+                body=(f"The AD group {name} is not in the PacketStation OU. Nobody gets its role until IT creates it."),
+                detail={"group": name},
+                condition=True,
+            )
+            raised["group_missing"] += 1
+        for name in groupmap.group_names():
+            if name.lower() not in missing:
+                jobs.resolve_event(conn, "group_missing", subject=name, title=f"Group missing from the OU: {name}")
+        text = report_text(res)
+        if text is not None:
+            jobs.raise_event(
+                conn,
+                "ad_reconcile_report",
+                subject=NAME,
+                title=text[0],
+                body=text[1],
+                detail=res.summary(),
+                severity="info",
+            )
+            raised["report"] = 1
+    return raised
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    jobs.reload_departments(conn)
+    try:
+        res = userdir.refresh_from_ad(conn, jobs.directory(ctx), run_id=ctx.run_id)
+    except Exception as e:
+        # AD down or the run refused: tell the report's recipients once, until a run succeeds again
+        try:
+            with conn.transaction():
+                jobs.raise_event(
+                    conn,
+                    "ad_reconcile_report",
+                    subject=FAILED_SUBJECT,
+                    title="AD reconcile did not run",
+                    body=f"The nightly AD reconcile stopped and changed nothing: {type(e).__name__}: {redact(str(e))}",
+                    condition=True,
+                )
+        except Exception:
+            ctx.log.exception("reconcile failure notice not raised")
+        raise
+    counts: Counts = res.as_dict()
+    try:
+        with conn.transaction():
+            jobs.resolve_event(conn, "ad_reconcile_report", subject=FAILED_SUBJECT, title="AD reconcile did not run")
+        notify(conn, res)
+    except Exception as e:  # a notification problem never undoes the reconcile
+        ctx.log.exception("reconcile notifications failed")
+        ctx.error(f"notifications: {type(e).__name__}: {e}", degrade=False)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/backup_check.py

@@ -0,0 +1,92 @@
+"""The backup check, daily at 07:15 site time.
+
+BACKUP_DIR is this stack's backup folder on the host (/var/backups/packetstation/<stack>/), mounted read-only.
+The host's backup script writes `*.dump` files and the `last-success` and `last-failure` markers there. The
+check raises the `backup_stale` condition (subject: the stack) when the newest dump is missing, empty or older
+than 26 hours, or when `last-failure` is newer than `last-success`. Otherwise it resolves it. It only reads.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, time
+from pathlib import Path
+
+from ...feed.config import FeedSettings
+from .. import jobs
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(7, 15)
+NAME = "backup-check"
+MAX_AGE_HOURS = 26.0
+SUCCESS = "last-success"
+FAILURE = "last-failure"
+
+
+def _mtime(p: Path) -> datetime | None:
+    try:
+        return datetime.fromtimestamp(p.stat().st_mtime, UTC)
+    except OSError:
+        return None
+
+
+def inspect(folder: Path, now: datetime | None = None) -> tuple[Counts, list[str]]:
+    """(counts, problems) for a backup folder. No problems means the backups are fine."""
+    now = now or datetime.now(UTC)
+    counts: Counts = {"folder": str(folder), "dumps": 0, "newest": None, "age_hours": None, "size": None}
+    if not folder.is_dir():
+        return counts, [f"The backup folder {folder} is not there."]
+    dumps: list[tuple[datetime, Path, int]] = []
+    for p in folder.glob("*.dump"):
+        try:
+            st = p.stat()
+        except OSError:
+            continue
+        if p.is_file():
+            dumps.append((datetime.fromtimestamp(st.st_mtime, UTC), p, st.st_size))
+    counts["dumps"] = len(dumps)
+    problems: list[str] = []
+    if not dumps:
+        problems.append("No backup dump is in the folder.")
+    else:
+        at, path, size = max(dumps, key=lambda d: (d[0], d[1].name))
+        age = (now - at).total_seconds() / 3600
+        counts.update(newest=path.name, newest_at=at.isoformat(), age_hours=round(age, 1), size=size)
+        if size == 0:
+            problems.append(f"The newest dump {path.name} is empty.")
+        if age > MAX_AGE_HOURS:
+            problems.append(f"The newest dump {path.name} is {age:.0f} hours old.")
+    ok_at, fail_at = _mtime(folder / SUCCESS), _mtime(folder / FAILURE)
+    counts["last_success"] = ok_at.isoformat() if ok_at else None
+    counts["last_failure"] = fail_at.isoformat() if fail_at else None
+    if fail_at is not None and (ok_at is None or fail_at > ok_at):
+        problems.append("The last backup run failed (last-failure is newer than last-success).")
+    return counts, problems
+
+
+def run(ctx: JobContext) -> Counts:
+    stack = str(ctx.settings.stack)
+    counts, problems = inspect(Path(ctx.settings.backup_dir))
+    counts["problems"] = problems
+    title = f"Backup failed or stale ({stack})"
+    with ctx.conn.transaction():
+        if problems:
+            jobs.raise_event(
+                ctx.conn,
+                "backup_stale",
+                subject=stack,
+                title=title,
+                body="\n".join(problems),
+                detail={k: v for k, v in counts.items() if k != "problems"},
+                severity="critical",
+                condition=True,
+            )
+        else:
+            jobs.resolve_event(ctx.conn, "backup_stale", subject=stack, title=title)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/integrity.py

@@ -0,0 +1,275 @@
+"""The integrity check, nightly at 04:00 site time.
+
+- Seals: every unbroken seal of a filed packet is verified again with the seal key.
+- Asset hashes: the sha256 of every file uploaded in the last 7 days, every night, and of every file on Sundays
+  (site time). Each check sets assets.hash_checked_at and hash_ok.
+- Missing files: every asset row whose file is not there.
+- Orphan files: files under ASSETS_DIR that no asset row names. Reported only, never deleted. A file younger than
+  an hour is skipped (an upload in flight).
+- The audit hash chain, from the first row.
+
+One integrity_findings row per finding (at most 500 per kind; the counts hold the whole number). Any finding
+raises the `integrity_failure` condition; a clean, complete run resolves it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import re
+import time as clock
+from datetime import datetime, time
+from pathlib import Path
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from ... import audit
+from ...feed.config import FeedSettings
+from ...settings import SecretFileError
+from .. import jobs
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(4, 0)
+NAME = "integrity"
+KINDS = ("seal", "asset_hash", "asset_missing", "blob_orphan", "audit_chain")
+CAP = 500
+RECENT_DAYS = 7
+ORPHAN_MIN_AGE_SECONDS = 3600
+CHUNK = 1024 * 1024
+SKIP_DIRS = {"lost+found"}
+SUBJECT = "integrity"
+LABELS = {
+    "seal": "seal does not verify",
+    "asset_hash": "file changed",
+    "asset_missing": "file missing",
+    "blob_orphan": "file with no asset row",
+    "audit_chain": "audit chain break",
+}
+
+
+class Findings:
+    """Collects findings and writes one row each, capped per kind."""
+
+    def __init__(self, conn: Any, run_id: int):
+        self.conn = conn
+        self.run_id = run_id
+        self.counts: dict[str, int] = dict.fromkeys(KINDS, 0)
+        self.samples: dict[str, list[str]] = {k: [] for k in KINDS}
+
+    def add(self, kind: str, subject: str, detail: dict[str, Any]) -> None:
+        self.counts[kind] += 1
+        if self.counts[kind] > CAP:
+            return
+        if len(self.samples[kind]) < 10:
+            self.samples[kind].append(subject)
+        self.conn.execute(
+            "INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, %s, %s, %s)",
+            (self.run_id, kind, subject, Jsonb(detail)),
+        )
+
+    @property
+    def total(self) -> int:
+        return sum(self.counts.values())
+
+
+# ---------------------------------------------------------------- seals
+
+
+def check_seals(ctx: JobContext, found: Findings) -> int:
+    """Verify every unbroken seal of a filed packet. Returns the number checked."""
+    from ...domain import packets
+    from ...domain.pipeline import seal_hash, seal_key_id
+
+    conn = ctx.conn
+    rows = conn.execute(
+        "SELECT s.id, s.packet_id, s.hash, s.key_id FROM packet_seals s JOIN packets p ON p.id = s.packet_id"
+        " WHERE s.broken_at IS NULL AND p.status = 'filed' ORDER BY s.packet_id"
+    ).fetchall()
+    if not rows:
+        return 0
+    try:
+        key = ctx.settings.seal_key()
+    except SecretFileError as e:
+        ctx.degrade(f"seals not checked: {e}")
+        return 0
+    kid = seal_key_id(key)
+    checked = 0
+    for seal_id, packet_id, digest, key_id in rows:
+        detail: dict[str, Any] = {"packet_id": packet_id, "seal_id": seal_id}
+        if key_id != kid:
+            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "sealed with another key", "key_id": key_id})
+            checked += 1
+            continue
+        try:
+            with conn.transaction():
+                row = packets.load_row(conn, packet_id)
+                content = packets.seal_content(conn, row) if row is not None else None
+        except Exception as e:  # one packet that cannot be read does not stop the others
+            ctx.error(f"packet {packet_id}: seal not checked: {type(e).__name__}: {e}")
+            continue
+        if content is None:
+            continue
+        ok = hmac.compare_digest(seal_hash(key, content), bytes(digest))
+        if not ok:
+            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "the packet no longer matches its seal"})
+        checked += 1
+    return checked
+
+
+# ---------------------------------------------------------------- assets
+
+
+def _path(root: Path, key: str) -> Path | None:
+    p = (root / key).resolve()
+    return p if p.is_relative_to(root) else None
+
+
+def sha256_of(path: Path) -> bytes:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.digest()
+
+
+def check_assets(ctx: JobContext, found: Findings, *, full: bool) -> dict[str, int]:
+    """Missing files for every asset row; the sha256 of the recent ones (all of them when full)."""
+    conn = ctx.conn
+    root = Path(ctx.settings.assets_dir).resolve()
+    rows = conn.execute(
+        "SELECT id, storage_key, sha256, size, uploaded_at >= now() - make_interval(days => %s) AS recent"
+        " FROM assets ORDER BY uploaded_at, id",
+        (RECENT_DAYS,),
+    ).fetchall()
+    hashed = 0
+    for aid, key, digest, size, recent in rows:
+        p = _path(root, str(key))
+        if p is None or not p.is_file():
+            found.add("asset_missing", str(aid), {"storage_key": key})
+            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
+            continue
+        if not (full or recent):
+            continue
+        try:
+            actual = sha256_of(p)
+        except OSError as e:
+            found.add("asset_missing", str(aid), {"storage_key": key, "error": e.strerror or type(e).__name__})
+            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
+            continue
+        ok = hmac.compare_digest(actual, bytes(digest))
+        hashed += 1
+        if not ok:
+            found.add(
+                "asset_hash",
+                str(aid),
+                {"storage_key": key, "expected": bytes(digest).hex(), "actual": actual.hex(), "size": size},
+            )
+        conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = %s WHERE id = %s", (ok, aid))
+    return {"assets": len(rows), "hashed": hashed}
+
+
+def check_orphans(ctx: JobContext, found: Findings) -> int:
+    """Files under ASSETS_DIR that no asset row names. Never deletes anything. Returns the files looked at."""
+    root = Path(ctx.settings.assets_dir)
+    if not root.is_dir():
+        ctx.degrade(f"ASSETS_DIR {root} is not a folder")
+        return 0
+    keys = {str(r[0]) for r in ctx.conn.execute("SELECT storage_key FROM assets")}
+    cutoff = clock.time() - ORPHAN_MIN_AGE_SECONDS
+    seen = 0
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
+        for name in filenames:
+            p = Path(dirpath) / name
+            rel = p.relative_to(root).as_posix()
+            seen += 1
+            if rel in keys:
+                continue
+            try:
+                st = p.stat()
+            except OSError:
+                continue
+            if st.st_mtime > cutoff:
+                continue
+            found.add(
+                "blob_orphan", rel, {"size": st.st_size, "mtime": datetime.fromtimestamp(st.st_mtime).isoformat()}
+            )
+    return seen
+
+
+# ---------------------------------------------------------------- audit chain
+
+_ROW = re.compile(r"^row (\d+):")
+
+
+def check_audit(ctx: JobContext, found: Findings) -> int:
+    ok, n, detail = audit.verify_chain(ctx.conn)
+    if not ok:
+        m = _ROW.match(detail)
+        found.add("audit_chain", m.group(1) if m else "?", {"detail": detail, "rows_checked": n})
+    return n
+
+
+# ---------------------------------------------------------------- the run
+
+
+def _event_text(found: Findings) -> tuple[str, str]:
+    parts = [f"{n} {LABELS[k]}" for k, n in found.counts.items() if n]
+    lines = []
+    for k, n in found.counts.items():
+        if n:
+            lines.append(f"{LABELS[k].capitalize()}: {n} ({', '.join(found.samples[k])}{', ...' if n > 10 else ''}).")
+    lines.append("Admin, Security tab: the full list of findings.")
+    title = f"Integrity check found {found.total} problem{'s' if found.total != 1 else ''}: " + ", ".join(parts)
+    return title, "\n".join(lines)
+
+
+def full_day(now_site: datetime) -> bool:
+    """Sunday, site time: every asset is rehashed."""
+    return now_site.weekday() == 6
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    full = full_day(ctx.now_site())
+    found = Findings(conn, ctx.run_id)
+    counts: Counts = {"full": full}
+    with conn.transaction():
+        counts["seals"] = check_seals(ctx, found)
+    with conn.transaction():
+        counts.update(check_assets(ctx, found, full=full))
+    with conn.transaction():
+        counts["files"] = check_orphans(ctx, found)
+    with conn.transaction():
+        counts["audit_rows"] = check_audit(ctx, found)
+    counts["findings"] = dict(found.counts)
+    counts["total"] = found.total
+    try:
+        with conn.transaction():
+            if found.total:
+                title, body = _event_text(found)
+                jobs.raise_event(
+                    conn,
+                    "integrity_failure",
+                    subject=SUBJECT,
+                    title=title,
+                    body=body,
+                    detail={"run_id": ctx.run_id, **found.counts},
+                    severity="critical",
+                    condition=True,
+                )
+            elif not ctx.degraded:
+                jobs.resolve_event(conn, "integrity_failure", subject=SUBJECT, title="Integrity check failure")
+    except Exception as e:
+        ctx.log.exception("integrity notification failed")
+        ctx.error(f"notification: {type(e).__name__}: {e}", degrade=False)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/notify_digest.py

@@ -0,0 +1,30 @@
+"""notify-digest, every 5 minutes: send the daily digests whose time has passed (see notify/digest.py). Not
+pausable."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from ...feed.config import FeedSettings
+from ...notify import digest
+from . import Counts, JobContext, JobSpec, directory, reload_departments
+
+EVERY_SECONDS = 5 * 60
+
+
+def run(ctx: JobContext) -> Counts:
+    has = ctx.conn.execute("SELECT EXISTS (SELECT 1 FROM notification_outbox WHERE sent_at IS NULL)").fetchone()[0]
+    if not has:
+        return {"due": 0}
+    reload_departments(ctx.conn)
+    counts = digest.run_digest(ctx.conn, ctx.settings, directory(ctx), ctx.now_site())
+    if counts.get("failed"):
+        ctx.degrade(str(counts.get("error") or f"{counts['failed']} digest messages failed"))
+    return counts
+
+
+def _cadence(_: FeedSettings, __: datetime) -> int:
+    return EVERY_SECONDS
+
+
+SPEC = JobSpec(name="notify-digest", run=run, cadence=_cadence, pausable=False)

server/packetstation/worker/jobs/notify_send.py

@@ -0,0 +1,67 @@
+"""notify-send, every 60 seconds: seed the rules, note a new build, run the watch checks (at most every 15 minutes)
+and deliver the events nobody has handled yet. Not pausable."""
+
+from __future__ import annotations
+
+import time as clock
+from datetime import datetime
+
+from ...feed.config import FeedSettings
+from ...notify import seed as seed_rules
+from ...notify import send, watch
+from . import Counts, JobContext, JobSpec, directory, reload_departments
+
+EVERY_SECONDS = 60
+WATCH_SECONDS = 15 * 60
+WATCH_KEY = "notify.watch_at"
+BUILD_KEY = "notify.build_noted"
+
+
+def _due(ctx: JobContext, key: str, seconds: float) -> bool:
+    """True once per `seconds` per worker process; marks the time when it says so."""
+    now = clock.monotonic()
+    with ctx.stack.lock:
+        last = ctx.stack.cache.get(key)
+        if last is not None and now - float(last) < seconds:
+            return False
+        ctx.stack.cache[key] = now
+    return True
+
+
+def run(ctx: JobContext) -> Counts:
+    conn, s = ctx.conn, ctx.settings
+    counts: Counts = {}
+    added = seed_rules(conn, s)
+    if added:
+        counts["seeded"] = len(added)
+    if _due(ctx, BUILD_KEY, float("inf")):
+        try:
+            if watch.note_build(conn, s.ps_build) is not None:
+                counts["version_deployed"] = s.ps_build
+        except Exception as e:
+            ctx.stack.drop(BUILD_KEY)  # try again on the next run
+            ctx.error(f"version check: {type(e).__name__}", degrade=False)
+    if _due(ctx, WATCH_KEY, WATCH_SECONDS):
+        for name, check in (("drawings", watch.check_drawings), ("change orders", watch.check_change_orders)):
+            try:
+                counts.update(check(conn))
+            except Exception as e:
+                ctx.log.exception("watch check %s failed", name)
+                ctx.error(f"watch {name}: {type(e).__name__}")
+    pending = conn.execute(
+        "SELECT EXISTS (SELECT 1 FROM notification_events WHERE dispatched_at IS NULL AND event_type IS NOT NULL)"
+    ).fetchone()[0]
+    if pending:
+        reload_departments(conn)
+        out = send.dispatch_pending(conn, s, directory(ctx))
+        counts.update(out)
+        if out.get("failed") or out.get("retry"):
+            ctx.degrade(f"{out.get('failed', 0)} failed, {out.get('retry', 0)} waiting for the directory")
+    return counts
+
+
+def _cadence(_: FeedSettings, __: datetime) -> int:
+    return EVERY_SECONDS
+
+
+SPEC = JobSpec(name="notify-send", run=run, cadence=_cadence, pausable=False)

server/packetstation/worker/jobs/session_refresh.py

@@ -0,0 +1,137 @@
+"""Session refresh, every `roles_refresh_minutes` (the effective security setting, 15 by default).
+
+For every user with a live session: read the account from AD by objectGUID (batched), re-resolve its groups
+(nested) and userAccountControl, rebuild `user_roles` and each live session's roles snapshot and
+roles_checked_at. A user disabled or gone in AD is marked inactive and their sessions end (`ad_disabled`,
+`ad_gone`); a user left with no role has their sessions ended (`roles_lost`). Every AD read happens before any
+write: AD unreachable makes the run degraded and nothing is ended.
+"""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from datetime import datetime
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from ... import audit
+from ...auth import directory as userdir
+from ...auth import groups as groupmap
+from ...auth.ldap import AdUser, LdapUnavailable
+from ...feed.config import FeedSettings
+from ...settings import Settings
+from .. import jobs
+from . import Counts, JobContext, JobSpec
+
+NAME = "session-refresh"
+DEFAULT_MINUTES = 15
+
+_lock = threading.Lock()
+_minutes = DEFAULT_MINUTES
+
+
+def refresh_minutes(conn: Any, settings: Settings) -> int:
+    """The effective roles_refresh_minutes: the Admin's security setting when that module is there, else Settings."""
+    try:
+        from ...auth import secsettings
+    except ImportError:
+        return int(settings.roles_refresh_minutes)
+    try:
+        return int(secsettings.effective(conn, settings).roles_refresh_minutes)
+    except Exception:
+        return int(settings.roles_refresh_minutes)
+
+
+def _set_minutes(value: int) -> None:
+    global _minutes
+    with _lock:
+        _minutes = max(1, int(value))
+
+
+def live_users(conn: Any) -> list[uuid.UUID]:
+    rows = conn.execute(
+        "SELECT DISTINCT user_id FROM sessions WHERE revoked_at IS NULL AND expires_at > now() ORDER BY user_id"
+    ).fetchall()
+    return [r[0] for r in rows]
+
+
+def _deactivate(conn: Any, uid: uuid.UUID, found: AdUser | None) -> int:
+    gone = found is None
+    row = conn.execute("SELECT sam_account, active FROM users WHERE id = %s", (uid,)).fetchone()
+    ended = userdir.mark_inactive(conn, uid, reason="ad_gone" if gone else "ad_disabled", gone=gone, audit_by=NAME)
+    if row is not None and row[1]:
+        audit.append(
+            conn,
+            action="ad.user_deactivated",
+            subject_type="user",
+            subject_id=uid,
+            detail={"reason": "gone" if gone else "disabled", "sam_account": row[0], "by": NAME},
+        )
+    return len(ended)
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    _set_minutes(refresh_minutes(conn, ctx.settings))
+    jobs.reload_departments(conn)
+    uids = live_users(conn)
+    counts: Counts = {
+        "users": len(uids),
+        "refreshed": 0,
+        "roles_changed": 0,
+        "deactivated": 0,
+        "roles_lost": 0,
+        "sessions_ended": 0,
+    }
+    if not uids:
+        return counts
+
+    # Every AD read first: a failure part way changes nothing
+    d = jobs.directory(ctx)
+    try:
+        found = d.users_by_guid(uids)
+        groups_of: dict[uuid.UUID, list[str]] = {}
+        for uid, u in found.items():
+            if u is not None and not u.disabled:
+                groups_of[uid] = sorted(d.groups_of(u.dn))
+    except LdapUnavailable as e:
+        ctx.degrade(f"AD unavailable, nothing changed: {e}")
+        counts["ad"] = "unavailable"
+        return counts
+
+    with conn.transaction():
+        for uid in uids:
+            u = found.get(uid)
+            if u is None or u.disabled:
+                counts["sessions_ended"] += _deactivate(conn, uid, u)
+                counts["deactivated"] += 1
+                continue
+            grp = groups_of[uid]
+            roles = groupmap.roles_from_groups(grp)
+            before = userdir.roles_of(conn, uid)
+            userdir.upsert_user(conn, u, groups=groupmap.mapped_groups(grp))
+            userdir.set_roles(conn, uid, roles)
+            if sorted(before) != sorted(roles):
+                counts["roles_changed"] += 1
+            if not roles:
+                counts["roles_lost"] += 1
+                counts["sessions_ended"] += len(userdir.end_sessions(conn, uid, "roles_lost", by=NAME))
+                continue
+            conn.execute(
+                "UPDATE sessions SET roles_snapshot = %s, roles_checked_at = now()"
+                " WHERE user_id = %s AND revoked_at IS NULL AND expires_at > now()",
+                (Jsonb({"roles": roles, "groups": grp}), uid),
+            )
+            counts["refreshed"] += 1
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    """Every roles_refresh_minutes. The value is read at each run, so a changed setting applies after the next."""
+    with _lock:
+        return _minutes * 60
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/runs.py

@@ -18,6 +18,7 @@ from psycopg.types.json import Jsonb
 
 from .. import guard
 from ..feed import config
+from ..notify import watch
 from .jobs import JobContext, JobSpec
 from .redaction import clip, redact
 
@@ -65,6 +66,21 @@ def _target(result: Any) -> str:
     return text.rsplit(" (", 1)[0] if text.endswith(")") else text
 
 
+def _notify_feed(sch: Scheduler, conn: Any, name: str, status: str, errors: list[str], run_id: int) -> None:
+    """feed_failed and feed_degraded after a feed or drawing run. Never fails the run."""
+    if name not in watch.FEED_JOBS:
+        return
+    try:
+        conn.execute("SELECT 1")  # raise_event never raises, so find a broken connection first
+        watch.feed_run(conn, name, status, list(errors), run_id)
+    except Exception:
+        try:
+            with sch.connect() as c2:
+                watch.feed_run(c2, name, status, list(errors), run_id)
+        except Exception as e:
+            log.error("job %s: can't raise the feed notification: %s", name, type(e).__name__)
+
+
 def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]:
     """Run `name` once and return its job_runs row. Never raises for a job failure; raises when the database
     cannot be reached at all (the caller logs it)."""
@@ -129,6 +145,7 @@ def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]:
             # The run's connection is broken: write the result on a fresh one
             with sch.connect() as c2:
                 _finish(c2, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target)
+        _notify_feed(sch, conn, name, status, errors, run_id)
         try:
             pending_after = bool(
                 conn.execute(