Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/packetstation/worker/jobs/__init__.py

@@ -1,7 +1,7 @@
 """The job registry and the shapes every job uses.
 
-Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. The feed and
-drawing jobs are imported lazily by `load_jobs`, so the worker runs before they exist.
+Jobs import `from packetstation.worker.jobs import JobContext, JobSpec, register` and expose `SPEC`. Every job but
+housekeeping is imported lazily by `load_jobs`, so a job module that cannot import is logged and left out.
 """
 
 from __future__ import annotations
@@ -97,7 +97,16 @@ class JobSpec:
 JOBS: dict[str, JobSpec] = {}
 _loaded = False
 _load_lock = threading.Lock()
-LAZY = ("visual_feed", "drawing_lookup")
+LAZY = (
+    "visual_feed",
+    "drawing_lookup",
+    "ad_reconcile",
+    "session_refresh",
+    "integrity",
+    "backup_check",
+    "notify_send",
+    "notify_digest",
+)
 
 
 def register(spec: JobSpec) -> JobSpec:
@@ -117,12 +126,53 @@ def load_jobs() -> dict[str, JobSpec]:
                 try:
                     m = importlib.import_module(f"{__name__}.{mod}")
                     register(m.SPEC)
-                except ImportError as e:
+                except (ImportError, AttributeError) as e:
                     log.warning("job module %s not loaded: %s", mod, redact(str(e)))
             _loaded = True
     return JOBS
 
 
+NOTIFY = __name__.rsplit(".", 2)[0] + ".notify"
+
+
+def directory(ctx: JobContext) -> Any:
+    """The worker's directory (one per process, in the stack cache). Tests put a fake under "directory"."""
+    from ...auth.ldap import Directory
+
+    return ctx.stack.cached("directory", lambda: Directory(ctx.settings))
+
+
+def reload_departments(conn: Any) -> None:
+    """Load the departments table into the group map, so a department the Admin added counts at once."""
+    from ...auth import groups
+
+    deps = groups.load_departments(conn)
+    if deps:
+        groups.use_departments(deps)
+
+
+def raise_event(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.raise_event, or a logged no-op when the notify package is not there."""
+    try:
+        real = importlib.import_module(NOTIFY).raise_event
+    except (ImportError, AttributeError) as e:
+        log.warning("event %s not raised, notify not available: %s", event_type, e)
+        return None
+    result: int | None = real(conn, event_type, **kw)
+    return result
+
+
+def resolve_event(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.resolve_event, or a logged no-op when the notify package is not there."""
+    try:
+        real = importlib.import_module(NOTIFY).resolve_event
+    except (ImportError, AttributeError) as e:
+        log.warning("event %s not resolved, notify not available: %s", event_type, e)
+        return None
+    result: int | None = real(conn, event_type, **kw)
+    return result
+
+
 def seconds_until(now_site: datetime, at: time) -> int:
     """Seconds from now_site to the next `at` (site wall clock), at least 1."""
     target = datetime.combine(now_site.date(), at, tzinfo=now_site.tzinfo)
@@ -140,7 +190,11 @@ __all__ = [
     "JobContext",
     "JobSpec",
     "Stack",
+    "directory",
     "load_jobs",
+    "raise_event",
     "register",
+    "reload_departments",
+    "resolve_event",
     "seconds_until",
 ]

server/packetstation/worker/jobs/ad_reconcile.py

@@ -0,0 +1,133 @@
+"""The AD reconcile, nightly at 01:30 site time and on request ("Refresh from AD now" on the Admin screen).
+
+It reloads the departments, then runs `auth.directory.refresh_from_ad`: list the PacketStation OU, enumerate the
+members of every mapped group (IN_CHAIN, paged at 500), upsert users on objectGUID, mark disabled or gone users
+inactive and end their sessions, end the sessions of users left with no role, and flag missing or duplicate
+employee IDs. The run's counts are the report the Admin screen shows.
+
+Then the notifications: one `group_missing` condition per mapped group not in the OU (resolved once the group is
+there), and one `ad_reconcile_report` when the run created or deactivated anyone, found extra groups in the OU or
+flagged employee IDs. AD unreachable fails the run and changes nothing.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, time
+from typing import Any
+
+from ...auth import directory as userdir
+from ...auth import groups as groupmap
+from ...feed.config import FeedSettings
+from .. import jobs
+from ..redaction import redact
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(1, 30)
+NAME = "ad-reconcile"
+FAILED_SUBJECT = "ad-reconcile:failed"
+BODY_NAMES = 50  # names listed per section in the report mail
+
+
+def _names(items: list[str]) -> str:
+    shown = ", ".join(items[:BODY_NAMES])
+    more = len(items) - BODY_NAMES
+    return shown + (f" and {more} more" if more > 0 else "")
+
+
+def report_text(res: userdir.RefreshResult) -> tuple[str, str] | None:
+    """(title, body) of the reconcile report, or None when there is nothing to report."""
+    parts: list[str] = []
+    lines: list[str] = []
+    if res.created:
+        parts.append(f"{res.created} added")
+        lines.append(f"Added: {_names(res.created_names)}.")
+    if res.deactivated:
+        parts.append(f"{res.deactivated} deactivated")
+        lines.append(f"Deactivated (disabled or gone in AD): {_names(res.deactivated_names)}.")
+    if res.groups_extra:
+        parts.append(f"{len(res.groups_extra)} extra group{'s' if len(res.groups_extra) != 1 else ''}")
+        lines.append(f"Groups in the OU that Packet Station does not map: {_names(res.groups_extra)}.")
+    if res.emp_missing:
+        parts.append(f"{len(res.emp_missing)} without an employee ID")
+        lines.append(f"Accounts without an employee ID: {_names(res.emp_missing)}.")
+    if res.emp_duplicate:
+        parts.append(f"{len(res.emp_duplicate)} duplicate employee ID{'s' if len(res.emp_duplicate) != 1 else ''}")
+        lines.append(f"Employee IDs on more than one account: {_names(res.emp_duplicate)}.")
+    if not parts:
+        return None
+    if res.sessions_ended:
+        lines.append(f"Sessions ended: {res.sessions_ended}.")
+    lines.append(f"Accounts seen in the mapped groups: {res.users_seen}.")
+    return "AD reconcile: " + ", ".join(parts), "\n".join(lines)
+
+
+def notify(conn: Any, res: userdir.RefreshResult) -> dict[str, int]:
+    """Raise and resolve the reconcile's events in one transaction. Returns what was raised."""
+    raised = {"group_missing": 0, "report": 0}
+    missing = {n.lower() for n in res.groups_missing}
+    with conn.transaction():
+        for name in res.groups_missing:
+            jobs.raise_event(
+                conn,
+                "group_missing",
+                subject=name,
+                title=f"Group missing from the OU: {name}",
+                body=(f"The AD group {name} is not in the PacketStation OU. Nobody gets its role until IT creates it."),
+                detail={"group": name},
+                condition=True,
+            )
+            raised["group_missing"] += 1
+        for name in groupmap.group_names():
+            if name.lower() not in missing:
+                jobs.resolve_event(conn, "group_missing", subject=name, title=f"Group missing from the OU: {name}")
+        text = report_text(res)
+        if text is not None:
+            jobs.raise_event(
+                conn,
+                "ad_reconcile_report",
+                subject=NAME,
+                title=text[0],
+                body=text[1],
+                detail=res.summary(),
+                severity="info",
+            )
+            raised["report"] = 1
+    return raised
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    jobs.reload_departments(conn)
+    try:
+        res = userdir.refresh_from_ad(conn, jobs.directory(ctx), run_id=ctx.run_id)
+    except Exception as e:
+        # AD down or the run refused: tell the report's recipients once, until a run succeeds again
+        try:
+            with conn.transaction():
+                jobs.raise_event(
+                    conn,
+                    "ad_reconcile_report",
+                    subject=FAILED_SUBJECT,
+                    title="AD reconcile did not run",
+                    body=f"The nightly AD reconcile stopped and changed nothing: {type(e).__name__}: {redact(str(e))}",
+                    condition=True,
+                )
+        except Exception:
+            ctx.log.exception("reconcile failure notice not raised")
+        raise
+    counts: Counts = res.as_dict()
+    try:
+        with conn.transaction():
+            jobs.resolve_event(conn, "ad_reconcile_report", subject=FAILED_SUBJECT, title="AD reconcile did not run")
+        notify(conn, res)
+    except Exception as e:  # a notification problem never undoes the reconcile
+        ctx.log.exception("reconcile notifications failed")
+        ctx.error(f"notifications: {type(e).__name__}: {e}", degrade=False)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/backup_check.py

@@ -0,0 +1,92 @@
+"""The backup check, daily at 07:15 site time.
+
+BACKUP_DIR is this stack's backup folder on the host (/var/backups/packetstation/<stack>/), mounted read-only.
+The host's backup script writes `*.dump` files and the `last-success` and `last-failure` markers there. The
+check raises the `backup_stale` condition (subject: the stack) when the newest dump is missing, empty or older
+than 26 hours, or when `last-failure` is newer than `last-success`. Otherwise it resolves it. It only reads.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, time
+from pathlib import Path
+
+from ...feed.config import FeedSettings
+from .. import jobs
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(7, 15)
+NAME = "backup-check"
+MAX_AGE_HOURS = 26.0
+SUCCESS = "last-success"
+FAILURE = "last-failure"
+
+
+def _mtime(p: Path) -> datetime | None:
+    try:
+        return datetime.fromtimestamp(p.stat().st_mtime, UTC)
+    except OSError:
+        return None
+
+
+def inspect(folder: Path, now: datetime | None = None) -> tuple[Counts, list[str]]:
+    """(counts, problems) for a backup folder. No problems means the backups are fine."""
+    now = now or datetime.now(UTC)
+    counts: Counts = {"folder": str(folder), "dumps": 0, "newest": None, "age_hours": None, "size": None}
+    if not folder.is_dir():
+        return counts, [f"The backup folder {folder} is not there."]
+    dumps: list[tuple[datetime, Path, int]] = []
+    for p in folder.glob("*.dump"):
+        try:
+            st = p.stat()
+        except OSError:
+            continue
+        if p.is_file():
+            dumps.append((datetime.fromtimestamp(st.st_mtime, UTC), p, st.st_size))
+    counts["dumps"] = len(dumps)
+    problems: list[str] = []
+    if not dumps:
+        problems.append("No backup dump is in the folder.")
+    else:
+        at, path, size = max(dumps, key=lambda d: (d[0], d[1].name))
+        age = (now - at).total_seconds() / 3600
+        counts.update(newest=path.name, newest_at=at.isoformat(), age_hours=round(age, 1), size=size)
+        if size == 0:
+            problems.append(f"The newest dump {path.name} is empty.")
+        if age > MAX_AGE_HOURS:
+            problems.append(f"The newest dump {path.name} is {age:.0f} hours old.")
+    ok_at, fail_at = _mtime(folder / SUCCESS), _mtime(folder / FAILURE)
+    counts["last_success"] = ok_at.isoformat() if ok_at else None
+    counts["last_failure"] = fail_at.isoformat() if fail_at else None
+    if fail_at is not None and (ok_at is None or fail_at > ok_at):
+        problems.append("The last backup run failed (last-failure is newer than last-success).")
+    return counts, problems
+
+
+def run(ctx: JobContext) -> Counts:
+    stack = str(ctx.settings.stack)
+    counts, problems = inspect(Path(ctx.settings.backup_dir))
+    counts["problems"] = problems
+    title = f"Backup failed or stale ({stack})"
+    with ctx.conn.transaction():
+        if problems:
+            jobs.raise_event(
+                ctx.conn,
+                "backup_stale",
+                subject=stack,
+                title=title,
+                body="\n".join(problems),
+                detail={k: v for k, v in counts.items() if k != "problems"},
+                severity="critical",
+                condition=True,
+            )
+        else:
+            jobs.resolve_event(ctx.conn, "backup_stale", subject=stack, title=title)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/integrity.py

@@ -0,0 +1,275 @@
+"""The integrity check, nightly at 04:00 site time.
+
+- Seals: every unbroken seal of a filed packet is verified again with the seal key.
+- Asset hashes: the sha256 of every file uploaded in the last 7 days, every night, and of every file on Sundays
+  (site time). Each check sets assets.hash_checked_at and hash_ok.
+- Missing files: every asset row whose file is not there.
+- Orphan files: files under ASSETS_DIR that no asset row names. Reported only, never deleted. A file younger than
+  an hour is skipped (an upload in flight).
+- The audit hash chain, from the first row.
+
+One integrity_findings row per finding (at most 500 per kind; the counts hold the whole number). Any finding
+raises the `integrity_failure` condition; a clean, complete run resolves it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import re
+import time as clock
+from datetime import datetime, time
+from pathlib import Path
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from ... import audit
+from ...feed.config import FeedSettings
+from ...settings import SecretFileError
+from .. import jobs
+from . import Counts, JobContext, JobSpec, seconds_until
+
+AT = time(4, 0)
+NAME = "integrity"
+KINDS = ("seal", "asset_hash", "asset_missing", "blob_orphan", "audit_chain")
+CAP = 500
+RECENT_DAYS = 7
+ORPHAN_MIN_AGE_SECONDS = 3600
+CHUNK = 1024 * 1024
+SKIP_DIRS = {"lost+found"}
+SUBJECT = "integrity"
+LABELS = {
+    "seal": "seal does not verify",
+    "asset_hash": "file changed",
+    "asset_missing": "file missing",
+    "blob_orphan": "file with no asset row",
+    "audit_chain": "audit chain break",
+}
+
+
+class Findings:
+    """Collects findings and writes one row each, capped per kind."""
+
+    def __init__(self, conn: Any, run_id: int):
+        self.conn = conn
+        self.run_id = run_id
+        self.counts: dict[str, int] = dict.fromkeys(KINDS, 0)
+        self.samples: dict[str, list[str]] = {k: [] for k in KINDS}
+
+    def add(self, kind: str, subject: str, detail: dict[str, Any]) -> None:
+        self.counts[kind] += 1
+        if self.counts[kind] > CAP:
+            return
+        if len(self.samples[kind]) < 10:
+            self.samples[kind].append(subject)
+        self.conn.execute(
+            "INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, %s, %s, %s)",
+            (self.run_id, kind, subject, Jsonb(detail)),
+        )
+
+    @property
+    def total(self) -> int:
+        return sum(self.counts.values())
+
+
+# ---------------------------------------------------------------- seals
+
+
+def check_seals(ctx: JobContext, found: Findings) -> int:
+    """Verify every unbroken seal of a filed packet. Returns the number checked."""
+    from ...domain import packets
+    from ...domain.pipeline import seal_hash, seal_key_id
+
+    conn = ctx.conn
+    rows = conn.execute(
+        "SELECT s.id, s.packet_id, s.hash, s.key_id FROM packet_seals s JOIN packets p ON p.id = s.packet_id"
+        " WHERE s.broken_at IS NULL AND p.status = 'filed' ORDER BY s.packet_id"
+    ).fetchall()
+    if not rows:
+        return 0
+    try:
+        key = ctx.settings.seal_key()
+    except SecretFileError as e:
+        ctx.degrade(f"seals not checked: {e}")
+        return 0
+    kid = seal_key_id(key)
+    checked = 0
+    for seal_id, packet_id, digest, key_id in rows:
+        detail: dict[str, Any] = {"packet_id": packet_id, "seal_id": seal_id}
+        if key_id != kid:
+            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "sealed with another key", "key_id": key_id})
+            checked += 1
+            continue
+        try:
+            with conn.transaction():
+                row = packets.load_row(conn, packet_id)
+                content = packets.seal_content(conn, row) if row is not None else None
+        except Exception as e:  # one packet that cannot be read does not stop the others
+            ctx.error(f"packet {packet_id}: seal not checked: {type(e).__name__}: {e}")
+            continue
+        if content is None:
+            continue
+        ok = hmac.compare_digest(seal_hash(key, content), bytes(digest))
+        if not ok:
+            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "the packet no longer matches its seal"})
+        checked += 1
+    return checked
+
+
+# ---------------------------------------------------------------- assets
+
+
+def _path(root: Path, key: str) -> Path | None:
+    p = (root / key).resolve()
+    return p if p.is_relative_to(root) else None
+
+
+def sha256_of(path: Path) -> bytes:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.digest()
+
+
+def check_assets(ctx: JobContext, found: Findings, *, full: bool) -> dict[str, int]:
+    """Missing files for every asset row; the sha256 of the recent ones (all of them when full)."""
+    conn = ctx.conn
+    root = Path(ctx.settings.assets_dir).resolve()
+    rows = conn.execute(
+        "SELECT id, storage_key, sha256, size, uploaded_at >= now() - make_interval(days => %s) AS recent"
+        " FROM assets ORDER BY uploaded_at, id",
+        (RECENT_DAYS,),
+    ).fetchall()
+    hashed = 0
+    for aid, key, digest, size, recent in rows:
+        p = _path(root, str(key))
+        if p is None or not p.is_file():
+            found.add("asset_missing", str(aid), {"storage_key": key})
+            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
+            continue
+        if not (full or recent):
+            continue
+        try:
+            actual = sha256_of(p)
+        except OSError as e:
+            found.add("asset_missing", str(aid), {"storage_key": key, "error": e.strerror or type(e).__name__})
+            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
+            continue
+        ok = hmac.compare_digest(actual, bytes(digest))
+        hashed += 1
+        if not ok:
+            found.add(
+                "asset_hash",
+                str(aid),
+                {"storage_key": key, "expected": bytes(digest).hex(), "actual": actual.hex(), "size": size},
+            )
+        conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = %s WHERE id = %s", (ok, aid))
+    return {"assets": len(rows), "hashed": hashed}
+
+
+def check_orphans(ctx: JobContext, found: Findings) -> int:
+    """Files under ASSETS_DIR that no asset row names. Never deletes anything. Returns the files looked at."""
+    root = Path(ctx.settings.assets_dir)
+    if not root.is_dir():
+        ctx.degrade(f"ASSETS_DIR {root} is not a folder")
+        return 0
+    keys = {str(r[0]) for r in ctx.conn.execute("SELECT storage_key FROM assets")}
+    cutoff = clock.time() - ORPHAN_MIN_AGE_SECONDS
+    seen = 0
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
+        for name in filenames:
+            p = Path(dirpath) / name
+            rel = p.relative_to(root).as_posix()
+            seen += 1
+            if rel in keys:
+                continue
+            try:
+                st = p.stat()
+            except OSError:
+                continue
+            if st.st_mtime > cutoff:
+                continue
+            found.add(
+                "blob_orphan", rel, {"size": st.st_size, "mtime": datetime.fromtimestamp(st.st_mtime).isoformat()}
+            )
+    return seen
+
+
+# ---------------------------------------------------------------- audit chain
+
+_ROW = re.compile(r"^row (\d+):")
+
+
+def check_audit(ctx: JobContext, found: Findings) -> int:
+    ok, n, detail = audit.verify_chain(ctx.conn)
+    if not ok:
+        m = _ROW.match(detail)
+        found.add("audit_chain", m.group(1) if m else "?", {"detail": detail, "rows_checked": n})
+    return n
+
+
+# ---------------------------------------------------------------- the run
+
+
+def _event_text(found: Findings) -> tuple[str, str]:
+    parts = [f"{n} {LABELS[k]}" for k, n in found.counts.items() if n]
+    lines = []
+    for k, n in found.counts.items():
+        if n:
+            lines.append(f"{LABELS[k].capitalize()}: {n} ({', '.join(found.samples[k])}{', ...' if n > 10 else ''}).")
+    lines.append("Admin, Security tab: the full list of findings.")
+    title = f"Integrity check found {found.total} problem{'s' if found.total != 1 else ''}: " + ", ".join(parts)
+    return title, "\n".join(lines)
+
+
+def full_day(now_site: datetime) -> bool:
+    """Sunday, site time: every asset is rehashed."""
+    return now_site.weekday() == 6
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    full = full_day(ctx.now_site())
+    found = Findings(conn, ctx.run_id)
+    counts: Counts = {"full": full}
+    with conn.transaction():
+        counts["seals"] = check_seals(ctx, found)
+    with conn.transaction():
+        counts.update(check_assets(ctx, found, full=full))
+    with conn.transaction():
+        counts["files"] = check_orphans(ctx, found)
+    with conn.transaction():
+        counts["audit_rows"] = check_audit(ctx, found)
+    counts["findings"] = dict(found.counts)
+    counts["total"] = found.total
+    try:
+        with conn.transaction():
+            if found.total:
+                title, body = _event_text(found)
+                jobs.raise_event(
+                    conn,
+                    "integrity_failure",
+                    subject=SUBJECT,
+                    title=title,
+                    body=body,
+                    detail={"run_id": ctx.run_id, **found.counts},
+                    severity="critical",
+                    condition=True,
+                )
+            elif not ctx.degraded:
+                jobs.resolve_event(conn, "integrity_failure", subject=SUBJECT, title="Integrity check failure")
+    except Exception as e:
+        ctx.log.exception("integrity notification failed")
+        ctx.error(f"notification: {type(e).__name__}: {e}", degrade=False)
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    return seconds_until(now_site, AT)
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/jobs/notify_digest.py

@@ -0,0 +1,30 @@
+"""notify-digest, every 5 minutes: send the daily digests whose time has passed (see notify/digest.py). Not
+pausable."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from ...feed.config import FeedSettings
+from ...notify import digest
+from . import Counts, JobContext, JobSpec, directory, reload_departments
+
+EVERY_SECONDS = 5 * 60
+
+
+def run(ctx: JobContext) -> Counts:
+    has = ctx.conn.execute("SELECT EXISTS (SELECT 1 FROM notification_outbox WHERE sent_at IS NULL)").fetchone()[0]
+    if not has:
+        return {"due": 0}
+    reload_departments(ctx.conn)
+    counts = digest.run_digest(ctx.conn, ctx.settings, directory(ctx), ctx.now_site())
+    if counts.get("failed"):
+        ctx.degrade(str(counts.get("error") or f"{counts['failed']} digest messages failed"))
+    return counts
+
+
+def _cadence(_: FeedSettings, __: datetime) -> int:
+    return EVERY_SECONDS
+
+
+SPEC = JobSpec(name="notify-digest", run=run, cadence=_cadence, pausable=False)

server/packetstation/worker/jobs/notify_send.py

@@ -0,0 +1,67 @@
+"""notify-send, every 60 seconds: seed the rules, note a new build, run the watch checks (at most every 15 minutes)
+and deliver the events nobody has handled yet. Not pausable."""
+
+from __future__ import annotations
+
+import time as clock
+from datetime import datetime
+
+from ...feed.config import FeedSettings
+from ...notify import seed as seed_rules
+from ...notify import send, watch
+from . import Counts, JobContext, JobSpec, directory, reload_departments
+
+EVERY_SECONDS = 60
+WATCH_SECONDS = 15 * 60
+WATCH_KEY = "notify.watch_at"
+BUILD_KEY = "notify.build_noted"
+
+
+def _due(ctx: JobContext, key: str, seconds: float) -> bool:
+    """True once per `seconds` per worker process; marks the time when it says so."""
+    now = clock.monotonic()
+    with ctx.stack.lock:
+        last = ctx.stack.cache.get(key)
+        if last is not None and now - float(last) < seconds:
+            return False
+        ctx.stack.cache[key] = now
+    return True
+
+
+def run(ctx: JobContext) -> Counts:
+    conn, s = ctx.conn, ctx.settings
+    counts: Counts = {}
+    added = seed_rules(conn, s)
+    if added:
+        counts["seeded"] = len(added)
+    if _due(ctx, BUILD_KEY, float("inf")):
+        try:
+            if watch.note_build(conn, s.ps_build) is not None:
+                counts["version_deployed"] = s.ps_build
+        except Exception as e:
+            ctx.stack.drop(BUILD_KEY)  # try again on the next run
+            ctx.error(f"version check: {type(e).__name__}", degrade=False)
+    if _due(ctx, WATCH_KEY, WATCH_SECONDS):
+        for name, check in (("drawings", watch.check_drawings), ("change orders", watch.check_change_orders)):
+            try:
+                counts.update(check(conn))
+            except Exception as e:
+                ctx.log.exception("watch check %s failed", name)
+                ctx.error(f"watch {name}: {type(e).__name__}")
+    pending = conn.execute(
+        "SELECT EXISTS (SELECT 1 FROM notification_events WHERE dispatched_at IS NULL AND event_type IS NOT NULL)"
+    ).fetchone()[0]
+    if pending:
+        reload_departments(conn)
+        out = send.dispatch_pending(conn, s, directory(ctx))
+        counts.update(out)
+        if out.get("failed") or out.get("retry"):
+            ctx.degrade(f"{out.get('failed', 0)} failed, {out.get('retry', 0)} waiting for the directory")
+    return counts
+
+
+def _cadence(_: FeedSettings, __: datetime) -> int:
+    return EVERY_SECONDS
+
+
+SPEC = JobSpec(name="notify-send", run=run, cadence=_cadence, pausable=False)

server/packetstation/worker/jobs/session_refresh.py

@@ -0,0 +1,137 @@
+"""Session refresh, every `roles_refresh_minutes` (the effective security setting, 15 by default).
+
+For every user with a live session: read the account from AD by objectGUID (batched), re-resolve its groups
+(nested) and userAccountControl, rebuild `user_roles` and each live session's roles snapshot and
+roles_checked_at. A user disabled or gone in AD is marked inactive and their sessions end (`ad_disabled`,
+`ad_gone`); a user left with no role has their sessions ended (`roles_lost`). Every AD read happens before any
+write: AD unreachable makes the run degraded and nothing is ended.
+"""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from datetime import datetime
+from typing import Any
+
+from psycopg.types.json import Jsonb
+
+from ... import audit
+from ...auth import directory as userdir
+from ...auth import groups as groupmap
+from ...auth.ldap import AdUser, LdapUnavailable
+from ...feed.config import FeedSettings
+from ...settings import Settings
+from .. import jobs
+from . import Counts, JobContext, JobSpec
+
+NAME = "session-refresh"
+DEFAULT_MINUTES = 15
+
+_lock = threading.Lock()
+_minutes = DEFAULT_MINUTES
+
+
+def refresh_minutes(conn: Any, settings: Settings) -> int:
+    """The effective roles_refresh_minutes: the Admin's security setting when that module is there, else Settings."""
+    try:
+        from ...auth import secsettings
+    except ImportError:
+        return int(settings.roles_refresh_minutes)
+    try:
+        return int(secsettings.effective(conn, settings).roles_refresh_minutes)
+    except Exception:
+        return int(settings.roles_refresh_minutes)
+
+
+def _set_minutes(value: int) -> None:
+    global _minutes
+    with _lock:
+        _minutes = max(1, int(value))
+
+
+def live_users(conn: Any) -> list[uuid.UUID]:
+    rows = conn.execute(
+        "SELECT DISTINCT user_id FROM sessions WHERE revoked_at IS NULL AND expires_at > now() ORDER BY user_id"
+    ).fetchall()
+    return [r[0] for r in rows]
+
+
+def _deactivate(conn: Any, uid: uuid.UUID, found: AdUser | None) -> int:
+    gone = found is None
+    row = conn.execute("SELECT sam_account, active FROM users WHERE id = %s", (uid,)).fetchone()
+    ended = userdir.mark_inactive(conn, uid, reason="ad_gone" if gone else "ad_disabled", gone=gone, audit_by=NAME)
+    if row is not None and row[1]:
+        audit.append(
+            conn,
+            action="ad.user_deactivated",
+            subject_type="user",
+            subject_id=uid,
+            detail={"reason": "gone" if gone else "disabled", "sam_account": row[0], "by": NAME},
+        )
+    return len(ended)
+
+
+def run(ctx: JobContext) -> Counts:
+    conn = ctx.conn
+    _set_minutes(refresh_minutes(conn, ctx.settings))
+    jobs.reload_departments(conn)
+    uids = live_users(conn)
+    counts: Counts = {
+        "users": len(uids),
+        "refreshed": 0,
+        "roles_changed": 0,
+        "deactivated": 0,
+        "roles_lost": 0,
+        "sessions_ended": 0,
+    }
+    if not uids:
+        return counts
+
+    # Every AD read first: a failure part way changes nothing
+    d = jobs.directory(ctx)
+    try:
+        found = d.users_by_guid(uids)
+        groups_of: dict[uuid.UUID, list[str]] = {}
+        for uid, u in found.items():
+            if u is not None and not u.disabled:
+                groups_of[uid] = sorted(d.groups_of(u.dn))
+    except LdapUnavailable as e:
+        ctx.degrade(f"AD unavailable, nothing changed: {e}")
+        counts["ad"] = "unavailable"
+        return counts
+
+    with conn.transaction():
+        for uid in uids:
+            u = found.get(uid)
+            if u is None or u.disabled:
+                counts["sessions_ended"] += _deactivate(conn, uid, u)
+                counts["deactivated"] += 1
+                continue
+            grp = groups_of[uid]
+            roles = groupmap.roles_from_groups(grp)
+            before = userdir.roles_of(conn, uid)
+            userdir.upsert_user(conn, u, groups=groupmap.mapped_groups(grp))
+            userdir.set_roles(conn, uid, roles)
+            if sorted(before) != sorted(roles):
+                counts["roles_changed"] += 1
+            if not roles:
+                counts["roles_lost"] += 1
+                counts["sessions_ended"] += len(userdir.end_sessions(conn, uid, "roles_lost", by=NAME))
+                continue
+            conn.execute(
+                "UPDATE sessions SET roles_snapshot = %s, roles_checked_at = now()"
+                " WHERE user_id = %s AND revoked_at IS NULL AND expires_at > now()",
+                (Jsonb({"roles": roles, "groups": grp}), uid),
+            )
+            counts["refreshed"] += 1
+    return counts
+
+
+def cadence(_: FeedSettings, now_site: datetime) -> int:
+    """Every roles_refresh_minutes. The value is read at each run, so a changed setting applies after the next."""
+    with _lock:
+        return _minutes * 60
+
+
+SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)

server/packetstation/worker/runs.py

@@ -18,6 +18,7 @@ from psycopg.types.json import Jsonb
 
 from .. import guard
 from ..feed import config
+from ..notify import watch
 from .jobs import JobContext, JobSpec
 from .redaction import clip, redact
 
@@ -65,6 +66,21 @@ def _target(result: Any) -> str:
     return text.rsplit(" (", 1)[0] if text.endswith(")") else text
 
 
+def _notify_feed(sch: Scheduler, conn: Any, name: str, status: str, errors: list[str], run_id: int) -> None:
+    """feed_failed and feed_degraded after a feed or drawing run. Never fails the run."""
+    if name not in watch.FEED_JOBS:
+        return
+    try:
+        conn.execute("SELECT 1")  # raise_event never raises, so find a broken connection first
+        watch.feed_run(conn, name, status, list(errors), run_id)
+    except Exception:
+        try:
+            with sch.connect() as c2:
+                watch.feed_run(c2, name, status, list(errors), run_id)
+        except Exception as e:
+            log.error("job %s: can't raise the feed notification: %s", name, type(e).__name__)
+
+
 def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]:
     """Run `name` once and return its job_runs row. Never raises for a job failure; raises when the database
     cannot be reached at all (the caller logs it)."""
@@ -129,6 +145,7 @@ def run_job(sch: Scheduler, name: str, trigger: str) -> dict[str, Any]:
             # The run's connection is broken: write the result on a fresh one
             with sch.connect() as c2:
                 _finish(c2, run_id, status=status, counts=counts, errors=errors, visual_target=visual_target)
+        _notify_feed(sch, conn, name, status, errors, run_id)
         try:
             pending_after = bool(
                 conn.execute(