Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/notify/__init__.py

@@ -0,0 +1,13 @@
+"""Notifications: the event catalog, raising and resolving events, routing, delivery and the digest.
+
+Everyone else calls `raise_event` and `resolve_event` inside their own transaction. Only the notify-send and
+notify-digest jobs (and the Admin's test send) talk to the mail relay.
+"""
+
+from __future__ import annotations
+
+from .catalog import EVENT_TYPES, SECURITY_TYPES, EventType
+from .events import raise_event, resolve_event
+from .seeding import seed
+
+__all__ = ["EVENT_TYPES", "SECURITY_TYPES", "EventType", "raise_event", "resolve_event", "seed"]

server/packetstation/notify/catalog.py

@@ -0,0 +1,82 @@
+"""The notification event types. One routing rule per type lives in `notification_rules`.
+
+A security type cannot be opted out of on Account settings, and the Admin cannot exclude anyone from it either.
+A department-scoped type carries the department of its packet, so it can go to that department's Supervisor group.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class EventType:
+    key: str
+    label: str
+    description: str
+    security: bool = False
+    dept_scoped: bool = False
+    param: str | None = None  # "hours" or "days"
+    param_default: int | None = None
+
+
+_TYPES = (
+    EventType(
+        "security_alert",
+        "Security alert",
+        "An account cooldown started, a PIN was disabled after five wrong tries, or a session ended after three"
+        " wrong unlock or signature PINs.",
+        security=True,
+    ),
+    EventType(
+        "signin_burst",
+        "Sign-in burst",
+        "The per-address sign-in limit tripped, or many sign-ins failed across accounts in five minutes.",
+        security=True,
+    ),
+    EventType(
+        "integrity_failure",
+        "Integrity check failure",
+        "The nightly integrity check found a broken seal, a changed or missing file, or a break in the audit chain.",
+        security=True,
+    ),
+    EventType("feed_failed", "Feed run failed", "A VISUAL feed or drawing lookup run failed."),
+    EventType("feed_degraded", "Feed run degraded", "A VISUAL feed run finished with errors."),
+    EventType(
+        "drawing_failing",
+        "Drawing lookup failing",
+        "Drawing lookups in M-Files have failed for more than the set number of hours.",
+        param="hours",
+        param_default=24,
+    ),
+    EventType(
+        "ad_reconcile_report",
+        "AD reconcile report",
+        "The nightly AD reconcile added or deactivated people, found extra groups, or flagged employee IDs.",
+    ),
+    EventType("group_missing", "Group missing from the OU", "A mapped PS group is not in the PacketStation OU."),
+    EventType("backup_stale", "Backup failed or stale", "The newest database backup is missing, empty, old or failed."),
+    EventType(
+        "co_waiting",
+        "Change order waiting",
+        "A change order has waited for review for more than the set number of days.",
+        dept_scoped=True,
+        param="days",
+        param_default=3,
+    ),
+    EventType(
+        "qc_reject",
+        "Hold-point or ITP reject",
+        "Q.C. rejected a packet, or an ITP line was rejected.",
+        dept_scoped=True,
+    ),
+    EventType("version_deployed", "Version deployed", "The worker started on a new build."),
+    EventType("assets_quota", "Assets storage warning", "The assets storage passed its warning threshold."),
+)
+
+EVENT_TYPES: dict[str, EventType] = {t.key: t for t in _TYPES}
+SECURITY_TYPES: frozenset[str] = frozenset(t.key for t in _TYPES if t.security)
+
+
+def is_known(event_type: str) -> bool:
+    return event_type in EVENT_TYPES

server/packetstation/notify/digest.py

@@ -0,0 +1,154 @@
+"""notify-digest: once a rule's digest time has passed today (site time), send what it queued before then.
+
+Recipients are resolved per event, then grouped per address: one message per address listing every event due for
+it, across rules. One notifications_sent row per message (event_type 'digest'). An outbox row is marked sent once
+every address it goes to has it; a failed send leaves it for the next run, and an address that already has an
+event is not sent it again.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, time, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from psycopg.rows import dict_row
+
+from ..settings import Settings
+from . import smtp
+from .routing import DIRECTORY_DOWN, DirectoryDown, NotifyDirectory, Recipient, Resolver, load_rules
+from .send import deliver, event_lines, footer, record
+
+log = logging.getLogger(__name__)
+
+DEFAULT_TIME = time(7, 0)
+BACKOFF = timedelta(minutes=30)
+
+
+def _due_rows(conn: Any, settings: Settings, rules: dict[str, Any], now: datetime) -> list[dict[str, Any]]:
+    tz = ZoneInfo(settings.site_tz)
+    pending = [r[0] for r in conn.execute("SELECT DISTINCT event_type FROM notification_outbox WHERE sent_at IS NULL")]
+    out: list[dict[str, Any]] = []
+    for et in pending:
+        rule = rules.get(et)
+        at = rule.digest_time if rule is not None else DEFAULT_TIME
+        cutoff = datetime.combine(now.astimezone(tz).date(), at, tzinfo=tz)
+        if now < cutoff:
+            continue
+        with conn.cursor(row_factory=dict_row) as cur:
+            out.extend(
+                dict(r)
+                for r in cur.execute(
+                    "SELECT o.id AS outbox_id, o.queued_at, e.* FROM notification_outbox o"
+                    " JOIN notification_events e ON e.id = o.event_id"
+                    " WHERE o.sent_at IS NULL AND o.event_type = %s AND o.queued_at < %s ORDER BY o.id",
+                    (et, cutoff),
+                ).fetchall()
+            )
+    return out
+
+
+def _already(conn: Any, event_ids: list[int]) -> set[tuple[int, str]]:
+    """(event id, address) pairs a digest already carried, sent or recorded in dry run."""
+    done: set[tuple[int, str]] = set()
+    for ids, recips in conn.execute(
+        "SELECT event_ids, recipients FROM notifications_sent WHERE digest AND result IN ('sent', 'dry_run')"
+        " AND event_ids && %s::bigint[]",
+        (event_ids,),
+    ).fetchall():
+        for rc in recips or []:
+            addr = str(rc.get("address") or "").lower()
+            done.update((int(i), addr) for i in ids or [])
+    return done
+
+
+def run_digest(
+    conn: Any, settings: Settings, directory: NotifyDirectory | None, now: datetime | None = None
+) -> dict[str, Any]:
+    """Send the due digests. Returns counts."""
+    tz = ZoneInfo(settings.site_tz)
+    now = now or datetime.now(tz)
+    last = conn.execute(
+        "SELECT result, at FROM notifications_sent WHERE digest AND NOT test ORDER BY id DESC LIMIT 1"
+    ).fetchone()
+    rules = load_rules(conn)
+    rows = _due_rows(conn, settings, rules, now)
+    if not rows:
+        return {"due": 0}
+    if last is not None and last[0] == "failed" and now - last[1] < BACKOFF:
+        return {"due": len(rows), "backoff": True}
+    resolver = Resolver(conn, settings, directory)
+    per_event: dict[int, list[Recipient]] = {}
+    try:
+        for r in rows:
+            rule = rules.get(str(r["event_type"]))
+            per_event[int(r["id"])] = resolver.resolve(rule, r.get("dept_code")) if rule and rule.enabled else []
+    except DirectoryDown:
+        ids = sorted({int(r["id"]) for r in rows})
+        subject = smtp.full_subject(settings, "Daily digest")
+        record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[],
+            result="failed",
+            error=DIRECTORY_DOWN,
+            event_ids=ids,
+            digest=True,
+        )
+        return {"due": len(rows), "failed": 1, "error": DIRECTORY_DOWN}
+    done = _already(conn, sorted(per_event))
+    by_addr: dict[str, tuple[Recipient, list[dict[str, Any]]]] = {}
+    seen_event: dict[str, set[int]] = {}
+    for r in rows:
+        eid = int(r["id"])
+        for rc in per_event[eid]:
+            key = rc.address.lower()
+            if (eid, key) in done or eid in seen_event.setdefault(key, set()):
+                continue
+            seen_event[key].add(eid)
+            by_addr.setdefault(key, (rc, []))[1].append(r)
+    failed_events: set[int] = set()
+    sent_by_event: dict[int, int] = {}
+    counts: dict[str, Any] = {"due": len(rows), "messages": 0}
+    for _key, (rc, events) in sorted(by_addr.items()):
+        n = len(events)
+        subject = smtp.full_subject(settings, f"Daily digest: {n} notification{'s' if n != 1 else ''}")
+        lines: list[str] = [f"{n} Packet Station notification{'s' if n != 1 else ''} since the last digest.", ""]
+        for i, ev in enumerate(events, 1):
+            block = event_lines(settings, ev)
+            lines.append(f"{i}. {block[0]}")
+            lines.extend(f"   {x}" if x else "" for x in block[1:])
+            lines.append("")
+        lines.extend(footer(settings))
+        ids = [int(e["id"]) for e in events]
+        result, error = deliver(settings, subject, "\n".join(lines), [rc])
+        sid = record(
+            conn,
+            event_type="digest",
+            subject=subject,
+            recipients=[rc],
+            result=result,
+            error=error,
+            event_ids=ids,
+            digest=True,
+        )
+        counts["messages"] += 1
+        counts[result] = counts.get(result, 0) + 1
+        if result in ("sent", "dry_run"):
+            for i in ids:
+                sent_by_event[i] = sid
+        else:
+            failed_events.update(ids)
+    with conn.transaction():
+        for r in rows:
+            eid = int(r["id"])
+            if eid in failed_events:
+                continue
+            conn.execute(
+                "UPDATE notification_outbox SET sent_at = now(), sent_id = %s WHERE id = %s AND sent_at IS NULL",
+                (sent_by_event.get(eid), r["outbox_id"]),
+            )
+    counts["left"] = len({int(r["id"]) for r in rows if int(r["id"]) in failed_events})
+    return counts

server/packetstation/notify/events.py

@@ -0,0 +1,134 @@
+"""Raise and resolve notification events inside the caller's transaction. No I/O here: notify-send mails them.
+
+A standing condition stays open (resolved_at null) until it clears. Raising it again while open only counts it.
+Clearing it closes the row and adds a recovery row that goes to the same rule. A one-shot is inserted resolved.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .catalog import EVENT_TYPES
+
+log = logging.getLogger(__name__)
+
+SEVERITIES = ("info", "warning", "critical")
+TITLE_MAX = 200
+BODY_MAX = 8000
+SUBJECT_MAX = 200
+
+
+def _jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value if value is not None else {}, default=str))
+
+
+def _clip(text: str | None, n: int) -> str:
+    t = (text or "").strip()
+    return t if len(t) <= n else t[: n - 1] + "…"
+
+
+def raise_event(
+    conn: Any,
+    event_type: str,
+    *,
+    subject: str,
+    title: str,
+    body: str = "",
+    detail: dict[str, Any] | None = None,
+    dept_code: str | None = None,
+    severity: str = "warning",
+    condition: bool = False,
+    dedupe_minutes: int = 0,
+) -> int | None:
+    """Record one event. Returns the new row's id, or None when it was deduped (or could not be recorded).
+
+    Never raises: a notification must not break a sign-in, a reject or a job. A failure is logged.
+    """
+    if event_type not in EVENT_TYPES:
+        log.error("raise_event: unknown event type %r; not recorded", event_type)
+        return None
+    if severity not in SEVERITIES:
+        severity = "warning"
+    subj = _clip(subject, SUBJECT_MAX)
+    args = (_clip(title, TITLE_MAX), _clip(body, BODY_MAX), Jsonb(_jsonable(detail)), dept_code)
+    try:
+        with conn.transaction():
+            if condition:
+                row = conn.execute(
+                    "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                    " dept_code) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
+                    " ON CONFLICT (kind, coalesce(subject, '')) WHERE resolved_at IS NULL DO UPDATE SET"
+                    " raised_count = notification_events.raised_count + 1, last_raised_at = now(),"
+                    " detail = EXCLUDED.detail, body = EXCLUDED.body"
+                    " RETURNING id, (xmax = 0) AS inserted",
+                    (event_type, event_type, severity, subj, *args),
+                ).fetchone()
+                return int(row[0]) if row is not None and row[1] else None
+            if dedupe_minutes > 0:
+                recent = conn.execute(
+                    "SELECT 1 FROM notification_events WHERE event_type = %s AND subject = %s"
+                    " AND created_at > now() - make_interval(mins => %s) LIMIT 1",
+                    (event_type, subj, int(dedupe_minutes)),
+                ).fetchone()
+                if recent is not None:
+                    return None
+            row = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now()) RETURNING id",
+                (event_type, event_type, severity, subj, *args),
+            ).fetchone()
+            return int(row[0])
+    except Exception:
+        log.exception("raise_event %s failed; not recorded", event_type)
+        return None
+
+
+def resolve_event(conn: Any, event_type: str, *, subject: str, title: str | None = None, body: str = "") -> int | None:
+    """Close the open condition for (event_type, subject) and add its recovery row. Returns the recovery row's id,
+    or None when nothing was open. Never raises."""
+    if event_type not in EVENT_TYPES:
+        log.error("resolve_event: unknown event type %r", event_type)
+        return None
+    subj = _clip(subject, SUBJECT_MAX)
+    try:
+        with conn.transaction():
+            row = conn.execute(
+                "UPDATE notification_events SET resolved_at = now() WHERE kind = %s AND coalesce(subject, '') = %s"
+                " AND resolved_at IS NULL RETURNING id, title, dept_code",
+                (event_type, subj),
+            ).fetchone()
+            if row is None:
+                return None
+            open_id, open_title, dept = row
+            base = title or open_title or EVENT_TYPES[event_type].label
+            rec = conn.execute(
+                "INSERT INTO notification_events (kind, event_type, severity, subject, title, body, detail,"
+                " dept_code, resolved_at, recovery_of) VALUES (%s, %s, 'info', %s, %s, %s, %s, %s, now(), %s)"
+                " RETURNING id",
+                (
+                    event_type,
+                    event_type,
+                    subj,
+                    _clip(f"Cleared: {base}", TITLE_MAX),
+                    _clip(body, BODY_MAX),
+                    Jsonb({"recovery_of": open_id}),
+                    dept,
+                    open_id,
+                ),
+            ).fetchone()
+            return int(rec[0])
+    except Exception:
+        log.exception("resolve_event %s failed", event_type)
+        return None
+
+
+def open_event(conn: Any, event_type: str, subject: str) -> int | None:
+    row = conn.execute(
+        "SELECT id FROM notification_events WHERE kind = %s AND coalesce(subject, '') = %s AND resolved_at IS NULL",
+        (event_type, subject),
+    ).fetchone()
+    return int(row[0]) if row else None

server/packetstation/notify/routing.py

@@ -0,0 +1,222 @@
+"""Rules and recipients. Recipients are resolved when a message goes, so a group stays current.
+
+group: the members of an AD group by DN (nested, enabled accounts with mail). user: users.email of an active user.
+mailbox: the address. dept_supervisor: the Supervisor group of the event's department, none without one.
+Exclusions drop a person by user id and by that person's email.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from dataclasses import dataclass, field
+from datetime import time
+from typing import Any, Protocol
+
+from psycopg.rows import dict_row
+
+from ..auth import groups as groupmap
+from ..auth.ldap import AdUser
+from ..settings import Settings
+from .catalog import EVENT_TYPES
+
+log = logging.getLogger(__name__)
+
+ADDRESS_RE = re.compile(r"^[^@\s<>,;\"']{1,64}@[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251})\.[A-Za-z]{2,63}$")
+DIRECTORY_DOWN = "directory unavailable"
+
+
+class NotifyDirectory(Protocol):
+    """The directory calls notifications use (a subset of auth.ldap.DirectoryProtocol)."""
+
+    def get_user_by_guid(self, guid: uuid.UUID) -> AdUser | None: ...
+    def groups_of(self, dn: str) -> list[str]: ...
+    def members_of_dn(self, group_dn: str) -> list[AdUser]: ...
+    def search_groups(self, q: str, limit: int = 25) -> list[dict[str, Any]]: ...
+
+
+class DirectoryDown(Exception):
+    """AD could not be read while resolving a group."""
+
+
+def address_ok(value: str) -> bool:
+    v = value.strip()
+    return len(v) <= 254 and ADDRESS_RE.match(v) is not None
+
+
+def cn_of(dn: str) -> str:
+    """The first CN of a DN ("CN=PS-Admins,OU=..." -> "PS-Admins"); the DN itself when it has none."""
+    first = dn.split(",", 1)[0].strip()
+    return first[3:].strip() if first[:3].lower() == "cn=" else dn
+
+
+def dn_under(dn: str, base: str) -> bool:
+    b = base.strip().lower().replace(" ", "")
+    return bool(b) and dn.strip().lower().replace(" ", "").endswith("," + b)
+
+
+def supervisor_group(dept_code: str | None) -> str | None:
+    """The Supervisor group name of a department, from the group map (None when it has none)."""
+    if not dept_code:
+        return None
+    for name, role in groupmap.current().entries:
+        if role.dept_code == dept_code and role.kind.endswith("supervisor"):
+            return name
+    return None
+
+
+def group_dn(settings: Settings, name: str) -> str:
+    base = settings.ldap_group_base_dn.strip()
+    return f"CN={name},{base}" if base else f"CN={name}"
+
+
+def supervisor_depts(roles: list[str]) -> list[str]:
+    """The departments a role set supervises (their Supervisor group), sorted."""
+    out: set[str] = set()
+    for text in roles:
+        try:
+            r = groupmap.parse_role(text)
+        except ValueError:
+            continue
+        if r.dept_code and r.kind.endswith("supervisor"):
+            out.add(r.dept_code)
+    return sorted(out)
+
+
+# ---------------------------------------------------------------- rules
+
+
+@dataclass
+class Rule:
+    event_type: str
+    enabled: bool
+    delivery: str
+    digest_time: time
+    param: int | None
+    version: int
+    recipients: list[dict[str, Any]] = field(default_factory=list)
+    exclusions: list[dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def param_value(self) -> int | None:
+        et = EVENT_TYPES.get(self.event_type)
+        if et is None or et.param is None:
+            return None
+        return self.param if self.param is not None else et.param_default
+
+
+def load_rules(conn: Any, event_type: str | None = None) -> dict[str, Rule]:
+    args = (event_type, event_type)
+    with conn.cursor(row_factory=dict_row) as cur:
+        rules = {
+            r["event_type"]: Rule(
+                r["event_type"],
+                bool(r["enabled"]),
+                str(r["delivery"]),
+                r["digest_time"],
+                r["param"],
+                int(r["version"]),
+            )
+            for r in cur.execute(
+                "SELECT event_type, enabled, delivery::text AS delivery, digest_time, param, version"
+                " FROM notification_rules WHERE %s::text IS NULL OR event_type = %s",
+                args,
+            ).fetchall()
+        }
+        for r in cur.execute(
+            "SELECT r.id, r.event_type, r.kind::text AS kind, r.group_dn, r.group_name, r.user_id, r.address,"
+            " u.display_name FROM notification_recipients r LEFT JOIN users u ON u.id = r.user_id"
+            " WHERE %s::text IS NULL OR r.event_type = %s ORDER BY r.id",
+            args,
+        ).fetchall():
+            if r["event_type"] in rules:
+                rules[r["event_type"]].recipients.append(dict(r))
+        for r in cur.execute(
+            "SELECT x.event_type, x.user_id, x.self, u.display_name, u.email FROM notification_exclusions x"
+            " LEFT JOIN users u ON u.id = x.user_id WHERE %s::text IS NULL OR x.event_type = %s"
+            " ORDER BY lower(u.display_name), x.user_id",
+            args,
+        ).fetchall():
+            if r["event_type"] in rules:
+                rules[r["event_type"]].exclusions.append(dict(r))
+    return rules
+
+
+def load_rule(conn: Any, event_type: str) -> Rule | None:
+    return load_rules(conn, event_type).get(event_type)
+
+
+# ---------------------------------------------------------------- resolving
+
+
+@dataclass
+class Recipient:
+    address: str
+    user_id: str | None
+    why: str
+
+    def as_json(self) -> dict[str, Any]:
+        return {"address": self.address, "user_id": self.user_id, "why": self.why}
+
+
+class Resolver:
+    """Resolves rules to addresses. Caches group members for its lifetime (one run or one request)."""
+
+    def __init__(self, conn: Any, settings: Settings, directory: NotifyDirectory | None):
+        self.conn = conn
+        self.settings = settings
+        self.directory = directory
+        self._members: dict[str, list[AdUser]] = {}
+
+    def members(self, dn: str) -> list[AdUser]:
+        key = dn.strip().lower()
+        if key not in self._members:
+            if self.directory is None:
+                raise DirectoryDown(DIRECTORY_DOWN)
+            try:
+                self._members[key] = list(self.directory.members_of_dn(dn))
+            except Exception as e:
+                log.warning("notify: can't read group %s: %s", cn_of(dn), type(e).__name__)
+                raise DirectoryDown(DIRECTORY_DOWN) from None
+        return self._members[key]
+
+    def _group(self, dn: str, why: str) -> list[Recipient]:
+        return [
+            Recipient(u.email.strip(), str(u.guid), why)
+            for u in self.members(dn)
+            if not u.disabled and u.email and u.email.strip()
+        ]
+
+    def resolve(self, rule: Rule, dept_code: str | None) -> list[Recipient]:
+        """Every address the rule reaches for an event of this department, after exclusions, one per address.
+
+        Raises DirectoryDown when a group cannot be read."""
+        found: list[Recipient] = []
+        for r in rule.recipients:
+            kind = r["kind"]
+            if kind == "group" and r.get("group_dn"):
+                found.extend(self._group(r["group_dn"], f"group {r.get('group_name') or cn_of(r['group_dn'])}"))
+            elif kind == "user" and r.get("user_id"):
+                row = self.conn.execute(
+                    "SELECT email FROM users WHERE id = %s AND active AND coalesce(email, '') <> ''", (r["user_id"],)
+                ).fetchone()
+                if row is not None:
+                    found.append(Recipient(str(row[0]).strip(), str(r["user_id"]), "user"))
+            elif kind == "mailbox" and r.get("address"):
+                found.append(Recipient(str(r["address"]).strip(), None, "mailbox"))
+            elif kind == "dept_supervisor":
+                name = supervisor_group(dept_code)
+                if name:
+                    found.extend(self._group(group_dn(self.settings, name), f"Supervisor group of {dept_code}"))
+        excluded_ids = {str(x["user_id"]) for x in rule.exclusions}
+        excluded_mail = {str(x["email"]).strip().lower() for x in rule.exclusions if x.get("email")}
+        out: list[Recipient] = []
+        seen: set[str] = set()
+        for rc in found:
+            key = rc.address.lower()
+            if key in seen or (rc.user_id and rc.user_id in excluded_ids) or key in excluded_mail:
+                continue
+            seen.add(key)
+            out.append(rc)
+        return out

server/packetstation/notify/seeding.py

@@ -0,0 +1,38 @@
+"""Seed the routing rules: every missing event type enabled, immediate, digest 07:00, to PS-Admins.
+
+Idempotent. An existing rule is never touched, so the Admin's choices stand.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..auth.groups import ADMIN_GROUP
+from ..settings import Settings
+from .catalog import EVENT_TYPES
+
+
+def admins_dn(settings: Settings) -> str:
+    base = settings.ldap_group_base_dn.strip()
+    return f"CN={ADMIN_GROUP},{base}" if base else f"CN={ADMIN_GROUP}"
+
+
+def seed(conn: Any, settings: Settings) -> list[str]:
+    """Insert the missing rules. Returns the event types it inserted."""
+    added: list[str] = []
+    with conn.transaction():
+        for key, et in EVENT_TYPES.items():
+            row = conn.execute(
+                "INSERT INTO notification_rules (event_type, enabled, delivery, digest_time, param)"
+                " VALUES (%s, true, 'immediate', '07:00', %s) ON CONFLICT (event_type) DO NOTHING RETURNING event_type",
+                (key, et.param_default),
+            ).fetchone()
+            if row is None:
+                continue
+            conn.execute(
+                "INSERT INTO notification_recipients (event_type, kind, group_dn, group_name)"
+                " VALUES (%s, 'group', %s, %s)",
+                (key, admins_dn(settings), ADMIN_GROUP),
+            )
+            added.append(key)
+    return added

server/packetstation/notify/send.py

@@ -0,0 +1,264 @@
+"""notify-send: deliver the events nobody has handled yet, and the Admin's test send.
+
+Per event: rule off -> dispatched `rule_off`; immediate -> one message to the resolved recipients, one
+notifications_sent row, dispatched with the result; digest -> an outbox row, dispatched `queued`.
+NOTIFY_DRY_RUN records `dry_run` and sends nothing, except a test send, which always goes.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+
+from ..settings import Settings
+from . import smtp
+from .catalog import EVENT_TYPES
+from .routing import DIRECTORY_DOWN, DirectoryDown, NotifyDirectory, Recipient, Resolver, load_rules
+
+log = logging.getLogger(__name__)
+
+MAX_DIRECTORY_TRIES = 3
+RETRY_PREFIX = "directory_retry:"
+BATCH = 200
+FOOTER = "Sent by Packet Station. Who gets this is set on the Admin page, Notifications tab."
+
+
+def record(
+    conn: Any,
+    *,
+    event_type: str,
+    subject: str,
+    recipients: Sequence[Recipient],
+    result: str,
+    error: str | None = None,
+    event_ids: Sequence[int] = (),
+    digest: bool = False,
+    test: bool = False,
+    requested_by: Any = None,
+) -> int:
+    row = conn.execute(
+        "INSERT INTO notifications_sent (event_type, event_ids, digest, test, subject, recipients, result, error,"
+        " requested_by) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id",
+        (
+            event_type,
+            list(event_ids),
+            digest,
+            test,
+            subject,
+            Jsonb([r.as_json() for r in recipients]),
+            result,
+            (error or None) and str(error)[:1000],
+            requested_by,
+        ),
+    ).fetchone()
+    return int(row[0])
+
+
+def deliver(
+    settings: Settings, subject: str, body: str, recipients: Sequence[Recipient], *, test: bool = False
+) -> tuple[str, str | None]:
+    """(result, error) for one message. No recipients -> no_recipients; dry run (not a test) -> dry_run."""
+    if not recipients:
+        return "no_recipients", None
+    if settings.notify_dry_run and not test:
+        return "dry_run", None
+    try:
+        note = smtp.send(settings, subject, body, [r.address for r in recipients])
+    except smtp.MailError as e:
+        log.warning("notify: mail not sent: %s", e)
+        return "failed", str(e)
+    return "sent", note
+
+
+def site_time(settings: Settings, at: datetime | None) -> str:
+    if at is None:
+        return ""
+    return at.astimezone(ZoneInfo(settings.site_tz)).strftime("%Y-%m-%d %H:%M")
+
+
+def label_of(event_type: str) -> str:
+    et = EVENT_TYPES.get(event_type)
+    return et.label if et else event_type
+
+
+def event_title(ev: dict[str, Any]) -> str:
+    return str(ev.get("title") or label_of(str(ev.get("event_type"))))
+
+
+def event_lines(settings: Settings, ev: dict[str, Any]) -> list[str]:
+    """The plain-text block for one event (a whole immediate message, or one entry in a digest)."""
+    lines = [event_title(ev)]
+    if ev.get("body"):
+        lines += ["", str(ev["body"]).rstrip()]
+    lines.append("")
+    lines.append(f"Event: {label_of(str(ev.get('event_type')))}")
+    if ev.get("subject"):
+        lines.append(f"About: {ev['subject']}")
+    if ev.get("dept_code"):
+        lines.append(f"Department: {ev['dept_code']}")
+    lines.append(f"When: {site_time(settings, ev.get('created_at'))} ({settings.site_tz})")
+    if int(ev.get("raised_count") or 1) > 1:
+        lines.append(f"Seen {ev['raised_count']} times, last {site_time(settings, ev.get('last_raised_at'))}")
+    return lines
+
+
+def footer(settings: Settings) -> list[str]:
+    return ["", "--", f"{FOOTER} Stack {settings.stack}, build {settings.ps_build}."]
+
+
+def immediate_text(settings: Settings, ev: dict[str, Any]) -> tuple[str, str]:
+    return smtp.full_subject(settings, event_title(ev)), "\n".join(event_lines(settings, ev) + footer(settings))
+
+
+def _tries(result: Any) -> int:
+    text = str(result or "")
+    if text.startswith(RETRY_PREFIX):
+        try:
+            return int(text[len(RETRY_PREFIX) :])
+        except ValueError:
+            return 0
+    return 0
+
+
+def dispatch_pending(conn: Any, settings: Settings, directory: NotifyDirectory | None) -> dict[str, int]:
+    """Handle every undispatched event once. Returns counts by outcome."""
+    counts: dict[str, int] = {}
+    ids = [
+        int(r[0])
+        for r in conn.execute(
+            "SELECT id FROM notification_events WHERE dispatched_at IS NULL AND event_type IS NOT NULL"
+            " ORDER BY id LIMIT %s",
+            (BATCH,),
+        ).fetchall()
+    ]
+    if not ids:
+        return counts
+    rules = load_rules(conn)
+    resolver = Resolver(conn, settings, directory)
+    for eid in ids:
+        with conn.transaction(), conn.cursor(row_factory=dict_row) as cur:
+            ev = cur.execute(
+                "SELECT * FROM notification_events WHERE id = %s AND dispatched_at IS NULL FOR UPDATE SKIP LOCKED",
+                (eid,),
+            ).fetchone()
+            if ev is None:
+                continue
+            outcome = _dispatch_one(conn, settings, resolver, rules.get(str(ev["event_type"])), dict(ev))
+        counts[outcome] = counts.get(outcome, 0) + 1
+    return counts
+
+
+def _done(conn: Any, eid: int, result: str) -> None:
+    conn.execute(
+        "UPDATE notification_events SET dispatched_at = now(), dispatch_result = %s WHERE id = %s", (result, eid)
+    )
+
+
+def _dispatch_one(conn: Any, settings: Settings, resolver: Resolver, rule: Any, ev: dict[str, Any]) -> str:
+    eid = int(ev["id"])
+    if rule is None:
+        _done(conn, eid, "no_rule")
+        return "no_rule"
+    if not rule.enabled:
+        _done(conn, eid, "rule_off")
+        return "rule_off"
+    if rule.delivery == "digest":
+        conn.execute("INSERT INTO notification_outbox (event_id, event_type) VALUES (%s, %s)", (eid, ev["event_type"]))
+        _done(conn, eid, "queued")
+        return "queued"
+    subject, body = immediate_text(settings, ev)
+    try:
+        recipients = resolver.resolve(rule, ev.get("dept_code"))
+    except DirectoryDown:
+        tries = _tries(ev.get("dispatch_result")) + 1
+        record(
+            conn,
+            event_type=ev["event_type"],
+            subject=subject,
+            recipients=[],
+            result="failed",
+            error=DIRECTORY_DOWN,
+            event_ids=[eid],
+        )
+        if tries >= MAX_DIRECTORY_TRIES:
+            _done(conn, eid, "failed")
+            return "failed"
+        conn.execute(
+            "UPDATE notification_events SET dispatch_result = %s WHERE id = %s", (f"{RETRY_PREFIX}{tries}", eid)
+        )
+        return "retry"
+    result, error = deliver(settings, subject, body, recipients)
+    record(
+        conn,
+        event_type=ev["event_type"],
+        subject=subject,
+        recipients=recipients,
+        result=result,
+        error=error,
+        event_ids=[eid],
+    )
+    _done(conn, eid, result)
+    return result
+
+
+# ---------------------------------------------------------------- test send
+
+
+def send_test(
+    conn: Any,
+    settings: Settings,
+    directory: NotifyDirectory | None,
+    event_type: str,
+    *,
+    to: str | None = None,
+    requested_by: Any = None,
+) -> dict[str, Any]:
+    """Send a test of one event type now, even in dry run. With `to`, only to that address."""
+    et = EVENT_TYPES[event_type]
+    subject = smtp.full_subject(settings, f"Test: {et.label}")
+    body = "\n".join(
+        [
+            f'This is a test of the Packet Station notification "{et.label}".',
+            "",
+            et.description,
+            "",
+            "Nothing has happened. An Admin sent this from the Notifications tab.",
+            *footer(settings),
+        ]
+    )
+    recipients: list[Recipient] = []
+    error: str | None = None
+    result: str | None = None
+    if to:
+        recipients = [Recipient(to.strip(), None, "test address")]
+    else:
+        rule = load_rules(conn, event_type).get(event_type)
+        try:
+            recipients = Resolver(conn, settings, directory).resolve(rule, None) if rule else []
+        except DirectoryDown:
+            result, error = "failed", DIRECTORY_DOWN
+    if result is None:
+        result, error = deliver(settings, subject, body, recipients, test=True)
+    sent_id = record(
+        conn,
+        event_type=event_type,
+        subject=subject,
+        recipients=recipients,
+        result=result,
+        error=error,
+        test=True,
+        requested_by=requested_by,
+    )
+    return {
+        "result": result,
+        "subject": subject,
+        "recipients": [r.as_json() for r in recipients],
+        "error": error,
+        "sent_id": sent_id,
+    }

server/packetstation/notify/smtp.py

@@ -0,0 +1,86 @@
+"""Send one message through the relay. STARTTLS verified against SMTP_CA_BUNDLE, never a plain-text fallback.
+
+To: is the From address; the real recipients go in the envelope only, so nobody sees the list. No credentials.
+"""
+
+from __future__ import annotations
+
+import smtplib
+import ssl
+from collections.abc import Sequence
+from email.message import EmailMessage
+from email.utils import formatdate, make_msgid
+
+from ..settings import Settings
+
+TIMEOUT_SECONDS = 15
+MAIL_OFF = "Mail is off: SMTP_HOST is empty."
+
+
+class MailError(Exception):
+    """The relay could not take the message. The text is safe to store and show (no secrets)."""
+
+
+def tls_context(settings: Settings) -> ssl.SSLContext:
+    """The same verified context the doctor uses: system trust plus SMTP_CA_BUNDLE when set."""
+    return ssl.create_default_context(cafile=str(settings.smtp_ca_bundle) if settings.smtp_ca_bundle else None)
+
+
+def subject_prefix(settings: Settings) -> str:
+    return "[Packet Station]" if settings.stack == "prod" else "[Packet Station TEST]"
+
+
+def full_subject(settings: Settings, title: str) -> str:
+    return f"{subject_prefix(settings)} {title}".strip()[:250]
+
+
+def build_message(settings: Settings, subject: str, body: str) -> EmailMessage:
+    msg = EmailMessage()
+    msg["From"] = settings.smtp_from
+    msg["To"] = settings.smtp_from
+    msg["Subject"] = subject
+    msg["Date"] = formatdate(localtime=False)
+    domain = settings.smtp_from.rpartition("@")[2] or None
+    msg["Message-ID"] = make_msgid(domain=domain)
+    msg["Auto-Submitted"] = "auto-generated"
+    msg.set_content(body or subject)
+    return msg
+
+
+def send(settings: Settings, subject: str, body: str, to: Sequence[str]) -> str | None:
+    """Deliver one message to the envelope addresses `to`. Raises MailError when nothing was delivered.
+
+    Returns None, or a note naming the addresses the relay refused when it took the message for the rest."""
+    host = (settings.smtp_host or "").strip()
+    if not host:
+        raise MailError(MAIL_OFF)
+    if not to:
+        raise MailError("No recipients.")
+    msg = build_message(settings, subject, body)
+    try:
+        with smtplib.SMTP(host, settings.smtp_port, timeout=TIMEOUT_SECONDS) as smtp:
+            smtp.ehlo()
+            if settings.smtp_starttls:
+                # starttls raises SMTPNotSupportedError when the relay does not offer it: no fallback
+                smtp.starttls(context=tls_context(settings))
+                smtp.ehlo()
+            refused = smtp.send_message(msg, from_addr=settings.smtp_from, to_addrs=list(to))
+    except ssl.SSLError as e:
+        raise MailError(f"TLS: {e.reason or e}") from None
+    except ssl.CertificateError as e:
+        raise MailError(f"TLS: {e}") from None
+    except smtplib.SMTPNotSupportedError:
+        raise MailError("The relay does not offer STARTTLS; not sent.") from None
+    except smtplib.SMTPRecipientsRefused as e:
+        raise MailError(f"Every recipient was refused ({len(e.recipients)}).") from None
+    except smtplib.SMTPResponseException as e:
+        raise MailError(f"SMTP {e.smtp_code}: {_text(e.smtp_error)}") from None
+    except (OSError, smtplib.SMTPException) as e:
+        raise MailError(f"{type(e).__name__}: {e}") from None
+    if refused:
+        return f"Refused for {len(refused)} of {len(to)} recipients: {', '.join(sorted(refused))[:300]}"
+    return None
+
+
+def _text(value: bytes | str) -> str:
+    return (value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value))[:300]

server/packetstation/notify/watch.py

@@ -0,0 +1,165 @@
+"""The standing conditions notify-send checks itself (at most every 15 minutes), and the version check.
+
+drawing_failing: drawing lookups failing for more than the rule's hours (subject `drawings`, the list in the body).
+co_waiting: a change order pending review for more than the rule's days (subject `co:<id>`, the packet's department).
+version_deployed: the worker runs a build other than the last one noted in the settings row `notify.last_build`.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from .catalog import EVENT_TYPES
+from .events import raise_event, resolve_event
+from .routing import load_rules
+
+log = logging.getLogger(__name__)
+
+BUILD_KEY = "notify.last_build"
+LIST_MAX = 50
+
+
+def _param(conn: Any, event_type: str) -> int:
+    rule = load_rules(conn, event_type).get(event_type)
+    value = rule.param_value if rule is not None else None
+    default = EVENT_TYPES[event_type].param_default or 1
+    return int(value or default)
+
+
+def check_drawings(conn: Any) -> dict[str, Any]:
+    hours = _param(conn, "drawing_failing")
+    rows = conn.execute(
+        "SELECT dwg, failing_since, fail_count, note FROM drawing_lookups"
+        " WHERE failing_since IS NOT NULL AND failing_since < now() - make_interval(hours => %s)"
+        " ORDER BY failing_since, dwg",
+        (hours,),
+    ).fetchall()
+    if not rows:
+        rid = resolve_event(conn, "drawing_failing", subject="drawings", body="Every drawing lookup works again.")
+        return {"drawings_failing": 0, "drawings_cleared": rid is not None}
+    n = len(rows)
+    lines = [f"{r[0]}: failing since {r[1]:%Y-%m-%d %H:%M} UTC, {r[2]} tries. {r[3] or ''}".rstrip() for r in rows]
+    if n > LIST_MAX:
+        lines = [*lines[:LIST_MAX], f"and {n - LIST_MAX} more"]
+    raise_event(
+        conn,
+        "drawing_failing",
+        subject="drawings",
+        title=f"{n} drawing lookup{'s' if n != 1 else ''} failing for more than {hours} hours",
+        body="\n".join(lines),
+        detail={"count": n, "hours": hours, "drawings": [r[0] for r in rows[:LIST_MAX]]},
+        condition=True,
+    )
+    return {"drawings_failing": n}
+
+
+def check_change_orders(conn: Any) -> dict[str, Any]:
+    days = _param(conn, "co_waiting")
+    rows = conn.execute(
+        "SELECT co.id, co.co_no, co.issued_at, p.id, p.dept_code, p.title, p.drawing, pr.number, u.lot,"
+        " floor(extract(epoch FROM now() - co.issued_at) / 86400)::int"
+        " FROM change_orders co JOIN packets p ON p.id = co.packet_id JOIN units u ON u.id = p.unit_id"
+        " JOIN projects pr ON pr.id = u.project_id"
+        " WHERE co.review = 'pending' AND p.removed_at IS NULL AND co.issued_at < now() - make_interval(days => %s)"
+        " ORDER BY co.id",
+        (days,),
+    ).fetchall()
+    waiting = {f"co:{r[0]}" for r in rows}
+    raised = 0
+    for co_id, co_no, _issued, pid, dept, title, drawing, number, lot, age in rows:
+        what = " ".join(x for x in (f"{number}-{lot}", dept, title or drawing or "") if x)
+        new = raise_event(
+            conn,
+            "co_waiting",
+            subject=f"co:{co_id}",
+            title=f"Change order CO-{co_no} waiting {age} days: {what}",
+            body=f"CO-{co_no} on packet {pid} ({what}) has waited for review for {age} days.",
+            detail={"co_id": co_id, "packet_id": pid, "days": age},
+            dept_code=dept,
+            condition=True,
+        )
+        raised += new is not None
+    cleared = 0
+    for (subject,) in conn.execute(
+        "SELECT subject FROM notification_events WHERE kind = 'co_waiting' AND resolved_at IS NULL"
+    ).fetchall():
+        if subject not in waiting:
+            cleared += (
+                resolve_event(conn, "co_waiting", subject=subject, body="Reviewed or no longer waiting.") is not None
+            )
+    return {"co_waiting": len(rows), "co_raised": raised, "co_cleared": cleared}
+
+
+def note_build(conn: Any, build: str) -> int | None:
+    """Raise version_deployed when `build` differs from the last build noted. Returns the event id or None."""
+    build = (build or "").strip()
+    if not build:
+        return None
+    with conn.transaction():
+        row = conn.execute("SELECT value FROM settings WHERE key = %s FOR UPDATE", (BUILD_KEY,)).fetchone()
+        before = (row[0] or {}).get("build") if row is not None else None
+        if before == build:
+            return None
+        if row is None:
+            conn.execute(
+                "INSERT INTO settings (key, value, version, updated_at) VALUES (%s, %s, 1, now())"
+                " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, version = settings.version + 1,"
+                " updated_at = now()",
+                (BUILD_KEY, Jsonb({"build": build})),
+            )
+        else:
+            conn.execute(
+                "UPDATE settings SET value = %s, version = version + 1, updated_at = now() WHERE key = %s",
+                (Jsonb({"build": build}), BUILD_KEY),
+            )
+        log.info("version deployed: %s (was %s)", build, before or "none")
+        return raise_event(
+            conn,
+            "version_deployed",
+            subject=f"build:{build}",
+            title=f"Version {build} deployed",
+            body=f"The worker started on build {build}. The build before was {before or 'not recorded'}.",
+            detail={"build": build, "before": before},
+            severity="info",
+        )
+
+
+FEED_JOBS = ("visual-feed", "drawing-lookup")
+
+
+def feed_run(conn: Any, job: str, status: str, errors: list[str], run_id: int | None = None) -> None:
+    """feed_failed and feed_degraded after a visual-feed or drawing-lookup run."""
+    if job not in FEED_JOBS or status not in ("ok", "degraded", "failed"):
+        return
+    first = errors[0] if errors else ""
+    text = "\n".join(errors[:10])
+    if status == "failed":
+        raise_event(
+            conn,
+            "feed_failed",
+            subject=job,
+            title=f"{job} run failed",
+            body=f"Run {run_id} failed.\n{text}".strip(),
+            detail={"job": job, "run_id": run_id, "error": first},
+            severity="critical",
+            condition=True,
+        )
+        return
+    resolve_event(conn, "feed_failed", subject=job, body=f"Run {run_id} finished {status}.")
+    if job != "visual-feed":
+        return
+    if status == "degraded":
+        raise_event(
+            conn,
+            "feed_degraded",
+            subject=job,
+            title=f"{job} run degraded",
+            body=f"Run {run_id} finished with errors.\n{text}".strip(),
+            detail={"job": job, "run_id": run_id, "error": first},
+            condition=True,
+        )
+    else:
+        resolve_event(conn, "feed_degraded", subject=job, body=f"Run {run_id} finished ok.")