Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/notify/digest.py

@@ -0,0 +1,154 @@
+"""notify-digest: once a rule's digest time has passed today (site time), send what it queued before then.
+
+Recipients are resolved per event, then grouped per address: one message per address listing every event due for
+it, across rules. One notifications_sent row per message (event_type 'digest'). An outbox row is marked sent once
+every address it goes to has it; a failed send leaves it for the next run, and an address that already has an
+event is not sent it again.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, time, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from psycopg.rows import dict_row
+
+from ..settings import Settings
+from . import smtp
+from .routing import DIRECTORY_DOWN, DirectoryDown, NotifyDirectory, Recipient, Resolver, load_rules
+from .send import deliver, event_lines, footer, record
+
+log = logging.getLogger(__name__)
+
+DEFAULT_TIME = time(7, 0)
+BACKOFF = timedelta(minutes=30)
+
+
+def _due_rows(conn: Any, settings: Settings, rules: dict[str, Any], now: datetime) -> list[dict[str, Any]]:
+    tz = ZoneInfo(settings.site_tz)
+    pending = [r[0] for r in conn.execute("SELECT DISTINCT event_type FROM notification_outbox WHERE sent_at IS NULL")]
+    out: list[dict[str, Any]] = []
+    for et in pending:
+        rule = rules.get(et)
+        at = rule.digest_time if rule is not None else DEFAULT_TIME
+        cutoff = datetime.combine(now.astimezone(tz).date(), at, tzinfo=tz)
+        if now < cutoff:
+            continue
+        with conn.cursor(row_factory=dict_row) as cur:
+            out.extend(
+                dict(r)
+                for r in cur.execute(
+                    "SELECT o.id AS outbox_id, o.queued_at, e.* FROM notification_outbox o"
+                    " JOIN notification_events e ON e.id = o.event_id"
+                    " WHERE o.sent_at IS NULL AND o.event_type = %s AND o.queued_at < %s ORDER BY o.id",
+                    (et, cutoff),
+                ).fetchall()
+            )
+    return out
+
+
+def _already(conn: Any, event_ids: list[int]) -> set[tuple[int, str]]:
+    """(event id, address) pairs a digest already carried, sent or recorded in dry run."""
+    done: set[tuple[int, str]] = set()
+    for ids, recips in conn.execute(
+        "SELECT event_ids, recipients FROM notifications_sent WHERE digest AND result IN ('sent', 'dry_run')"
+        " AND event_ids && %s::bigint[]",
+        (event_ids,),
+    ).fetchall():
+        for rc in recips or []:
+            addr = str(rc.get("address") or "").lower()
+            done.update((int(i), addr) for i in ids or [])
+    return done
+
+
+def run_digest(
+    conn: Any, settings: Settings, directory: NotifyDirectory | None, now: datetime | None = None
+) -> dict[str, Any]:
+    """Send the due digests. Returns counts."""
+    tz = ZoneInfo(settings.site_tz)
+    now = now or datetime.now(tz)
+    last = conn.execute(
+        "SELECT result, at FROM notifications_sent WHERE digest AND NOT test ORDER BY id DESC LIMIT 1"
+    ).fetchone()
+    rules = load_rules(conn)
+    rows = _due_rows(conn, settings, rules, now)
+    if not rows:
+        return {"due": 0}
+    if last is not None and last[0] == "failed" and now - last[1] < BACKOFF:
+        return {"due": len(rows), "backoff": True}
+    resolver = Resolver(conn, settings, directory)
+    per_event: dict[int, list[Recipient]] = {}
+    try:
+        for r in rows:
+            rule = rules.get(str(r["event_type"]))
+            per_event[int(r["id"])] = resolver.resolve(rule, r.get("dept_code")) if rule and rule.enabled else []
+    except DirectoryDown:
+        ids = sorted({int(r["id"]) for r in rows})
+        subject = smtp.full_subject(settings, "Daily digest")
+        record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[],
+            result="failed",
+            error=DIRECTORY_DOWN,
+            event_ids=ids,
+            digest=True,
+        )
+        return {"due": len(rows), "failed": 1, "error": DIRECTORY_DOWN}
+    done = _already(conn, sorted(per_event))
+    by_addr: dict[str, tuple[Recipient, list[dict[str, Any]]]] = {}
+    seen_event: dict[str, set[int]] = {}
+    for r in rows:
+        eid = int(r["id"])
+        for rc in per_event[eid]:
+            key = rc.address.lower()
+            if (eid, key) in done or eid in seen_event.setdefault(key, set()):
+                continue
+            seen_event[key].add(eid)
+            by_addr.setdefault(key, (rc, []))[1].append(r)
+    failed_events: set[int] = set()
+    sent_by_event: dict[int, int] = {}
+    counts: dict[str, Any] = {"due": len(rows), "messages": 0}
+    for _key, (rc, events) in sorted(by_addr.items()):
+        n = len(events)
+        subject = smtp.full_subject(settings, f"Daily digest: {n} notification{'s' if n != 1 else ''}")
+        lines: list[str] = [f"{n} Packet Station notification{'s' if n != 1 else ''} since the last digest.", ""]
+        for i, ev in enumerate(events, 1):
+            block = event_lines(settings, ev)
+            lines.append(f"{i}. {block[0]}")
+            lines.extend(f"   {x}" if x else "" for x in block[1:])
+            lines.append("")
+        lines.extend(footer(settings))
+        ids = [int(e["id"]) for e in events]
+        result, error = deliver(settings, subject, "\n".join(lines), [rc])
+        sid = record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[rc],
+            result=result,
+            error=error,
+            event_ids=ids,
+            digest=True,
+        )
+        counts["messages"] += 1
+        counts[result] = counts.get(result, 0) + 1
+        if result in ("sent", "dry_run"):
+            for i in ids:
+                sent_by_event[i] = sid
+        else:
+            failed_events.update(ids)
+    with conn.transaction():
+        for r in rows:
+            eid = int(r["id"])
+            if eid in failed_events:
+                continue
+            conn.execute(
+                "UPDATE notification_outbox SET sent_at = now(), sent_id = %s WHERE id = %s AND sent_at IS NULL",
+                (sent_by_event.get(eid), r["outbox_id"]),
+            )
+    counts["left"] = len({int(r["id"]) for r in rows if int(r["id"]) in failed_events})
+    return counts