Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/notify/events.py

@@ -0,0 +1,134 @@
+"""Raise and resolve notification events inside the caller's transaction. No I/O here: notify-send mails them.
+
+A standing condition stays open (resolved_at null) until it clears. Raising it again while open only counts it.
+Clearing it closes the row and adds a recovery row that goes to the same rule. A one-shot is inserted resolved.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .catalog import EVENT_TYPES
+
+log = logging.getLogger(__name__)
+
+SEVERITIES = ("info", "warning", "critical")
+TITLE_MAX = 200
+BODY_MAX = 8000
+SUBJECT_MAX = 200
+
+
+def _jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value if value is not None else {}, default=str))
+
+
+def _clip(text: str | None, n: int) -> str:
+    t = (text or "").strip()
+    return t if len(t) <= n else t[: n - 1] + "…"
+
+
+def raise_event(
+    conn: Any,
+    event_type: str,
+    *,
+    subject: str,
+    title: str,
+    body: str = "",
+    detail: dict[str, Any] | None = None,
+    dept_code: str | None = None,
+    severity: str = "warning",
+    condition: bool = False,
+    dedupe_minutes: int = 0,
+) -> int | None:
+    """Record one event. Returns the new row's id, or None when it was deduped (or could not be recorded).
+
+    Never raises: a notification must not break a sign-in, a reject or a job. A failure is logged.
+    """
+    if event_type not in EVENT_TYPES:
+        log.error("raise_event: unknown event type %r; not recorded", event_type)
+        return None
+    if severity not in SEVERITIES:
+        severity = "warning"
+    subj = _clip(subject, SUBJECT_MAX)
+    args = (_clip(title, TITLE_MAX), _clip(body, BODY_MAX), Jsonb(_jsonable(detail)), dept_code)
+    try:
+        with conn.transaction():
+            if condition:
+                row = conn.execute(
+                    "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                    " dept_code) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
+                    " ON CONFLICT (kind, coalesce(subject, '')) WHERE resolved_at IS NULL DO UPDATE SET"
+                    " raised_count = notification_events.raised_count + 1, last_raised_at = now(),"
+                    " detail = EXCLUDED.detail, body = EXCLUDED.body"
+                    " RETURNING id, (xmax = 0) AS inserted",
+                    (event_type, event_type, severity, subj, *args),
+                ).fetchone()
+                return int(row[0]) if row is not None and row[1] else None
+            if dedupe_minutes > 0:
+                recent = conn.execute(
+                    "SELECT 1 FROM notification_events WHERE event_type = %s AND subject = %s"
+                    " AND created_at > now() - make_interval(mins => %s) LIMIT 1",
+                    (event_type, subj, int(dedupe_minutes)),
+                ).fetchone()
+                if recent is not None:
+                    return None
+            row = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now()) RETURNING id",
+                (event_type, event_type, severity, subj, *args),
+            ).fetchone()
+            return int(row[0])
+    except Exception:
+        log.exception("raise_event %s failed; not recorded", event_type)
+        return None
+
+
+def resolve_event(conn: Any, event_type: str, *, subject: str, title: str | None = None, body: str = "") -> int | None:
+    """Close the open condition for (event_type, subject) and add its recovery row. Returns the recovery row's id,
+    or None when nothing was open. Never raises."""
+    if event_type not in EVENT_TYPES:
+        log.error("resolve_event: unknown event type %r", event_type)
+        return None
+    subj = _clip(subject, SUBJECT_MAX)
+    try:
+        with conn.transaction():
+            row = conn.execute(
+                "UPDATE notification_events SET resolved_at = now() WHERE kind = %s AND coalesce(subject, '') = %s"
+                " AND resolved_at IS NULL RETURNING id, title, dept_code",
+                (event_type, subj),
+            ).fetchone()
+            if row is None:
+                return None
+            open_id, open_title, dept = row
+            base = title or open_title or EVENT_TYPES[event_type].label
+            rec = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at, recovery_of) VALUES (%s, %s, 'info', %s, %s, %s, %s, %s, now(), %s)"
+                " RETURNING id",
+                (
+                    event_type,
+                    event_type,
+                    subj,
+                    _clip(f"Cleared: {base}", TITLE_MAX),
+                    _clip(body, BODY_MAX),
+                    Jsonb({"recovery_of": open_id}),
+                    dept,
+                    open_id,
+                ),
+            ).fetchone()
+            return int(rec[0])
+    except Exception:
+        log.exception("resolve_event %s failed", event_type)
+        return None
+
+
+def open_event(conn: Any, event_type: str, subject: str) -> int | None:
+    row = conn.execute(
+        "SELECT id FROM notification_events WHERE kind = %s AND coalesce(subject, '') = %s AND resolved_at IS NULL",
+        (event_type, subject),
+    ).fetchone()
+    return int(row[0]) if row else None