Phase 9: Admin completion, reconcile, notifications, integrity, backups

Admin: Departments (name and stations; add shows the three groups IT must create), Profiles (welder stamp, station limits, active) with a CSV import that previews and applies only when clean, Checklists (weld and plain), Security (session, lockout and rate-limit settings with bounds, alerts for the last 24 hours, the integrity result), Access log with filters and a guarded CSV, Who is in, Stamp check, Notifications, and the employeeID flags. Every Admin change is audited. Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh (15 minutes), integrity (seals, asset hashes, missing and orphan files, the audit chain), backup-check, notify-send and notify-digest. Notifications route per event type to AD groups, users, mailboxes or the department Supervisor group, immediately or in a daily digest, over STARTTLS verified against the PESCO CA, with dry run on the test stack. Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21. CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19. Migration 0010. Closes the app part of I6, M3, O1, O2 and S3. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-21 18:27:09 -06:00
parent ce28d2f3c6
commit a82e056b6a
116 changed files with 15125 additions and 297 deletions
@@ -0,0 +1,321 @@
+"""Security alerts: raising them on a failed sign-in, a disabled PIN or an ended session, auditing policy denials,
+and the Admin Security tab's list of what happened.
+
+Raising goes through `notify.raise_event` (security_alert, signin_burst) inside the caller's database work. A
+failure here is logged and never breaks the sign-in or the request it rides on.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from datetime import timedelta
+from typing import Any
+
+from psycopg.rows import dict_row
+
+from .. import audit
+from ..auth import ratelimit, secsettings
+from ..settings import Settings
+
+log = logging.getLogger(__name__)
+
+SECURITY_ALERT = "security_alert"
+SIGNIN_BURST = "signin_burst"
+ALERT_DEDUPE_MINUTES = 15
+BURST_DEDUPE_MINUTES = 30
+BURST_KINDS = ("password", "pin")
+PIN_FAILURES_MIN = 3
+ALERTS_MAX = 1000
+DENIED_DEDUPE_SECONDS = 60.0
+DENIED_SKIP_CODES = frozenset({"wrong_secret"})
+
+ALERT_KINDS = (
+    "lockout",
+    "pin_failures",
+    "denied",
+    "stepup_wrong_pin",
+    "admin_change",
+    "it_edit",
+    "integrity",
+    "signin_burst",
+)
+
+
+def notify(conn: Any, event_type: str, **kw: Any) -> int | None:
+    """notify.raise_event in a savepoint. A failure is logged and returns None."""
+    try:
+        from ..notify import raise_event
+
+        with conn.transaction():
+            result: int | None = raise_event(conn, event_type, **kw)
+            return result
+    except Exception:
+        log.exception("could not raise %s", event_type)
+        return None
+
+
+def _recent_audit(conn: Any, action: str, subject: str, minutes: int) -> bool:
+    row = conn.execute(
+        "SELECT 1 FROM audit_events WHERE action = %s AND subject_type = 'security' AND subject_id = %s"
+        " AND at > now() - make_interval(mins => %s::int) LIMIT 1",
+        (action, subject, max(1, minutes)),
+    ).fetchone()
+    return row is not None
+
+
+def _security_audit(
+    conn: Any, action: str, subject: str, detail: dict[str, Any], *, user_id: Any = None, ip: str | None = None
+) -> None:
+    audit.append(
+        conn, action=action, user_id=user_id, subject_type="security", subject_id=subject, detail=detail, ip=ip
+    )
+
+
+def after_failure(
+    conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any = None
+) -> None:
+    """Call after a failed attempt was recorded. Raises security_alert when the account's cooldown starts, and
+    signin_burst when the address limit trips or the failures across accounts reach the burst threshold."""
+    try:
+        with conn.transaction():
+            _after_failure(conn, settings, account=account, ip=ip, kind=kind, user_id=user_id)
+    except Exception:
+        log.exception("security alert check failed")
+
+
+def _after_failure(conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any) -> None:
+    sec = secsettings.effective(conn, settings)
+    s = sec.settings or settings
+    now = ratelimit.db_now(conn)
+    if account:
+        wait = ratelimit.check(conn, s, accounts=[account], ip=None, now=now)
+        subject = f"account:{account}"
+        if wait is not None and not _recent_audit(conn, "auth.lockout", subject, sec.signin_account_cooldown_minutes):
+            minutes = max(1, (wait + 59) // 60)
+            _security_audit(
+                conn,
+                "auth.lockout",
+                subject,
+                {"account": account, "kind": kind, "minutes": minutes},
+                user_id=user_id,
+                ip=ip,
+            )
+            notify(
+                conn,
+                SECURITY_ALERT,
+                subject=subject,
+                title=f"Sign-in cooldown started for {account}",
+                body=(
+                    f"{sec.signin_account_limit} failed attempts on {account} within "
+                    f"{sec.signin_account_window_minutes} minutes. Further attempts are refused for {minutes} "
+                    f"minutes. Last address: {ip or 'unknown'}."
+                ),
+                detail={"account": account, "kind": kind, "ip": ip, "minutes": minutes},
+                dedupe_minutes=ALERT_DEDUPE_MINUTES,
+            )
+    if ip:
+        wait = ratelimit.check(conn, s, accounts=[], ip=ip, now=now)
+        subject = f"ip:{ip}"
+        if wait is not None and not _recent_audit(conn, "auth.signin_burst", subject, BURST_DEDUPE_MINUTES):
+            _security_audit(conn, "auth.signin_burst", subject, {"reason": "ip_limit", "ip": ip}, ip=ip)
+            notify(
+                conn,
+                SIGNIN_BURST,
+                subject=subject,
+                title=f"Sign-in limit reached for address {ip}",
+                body=(
+                    f"{sec.signin_ip_limit} failed sign-ins from {ip} within {sec.signin_ip_window_minutes} minutes. "
+                    "The address is refused until the count drops."
+                ),
+                detail={"reason": "ip_limit", "ip": ip},
+                dedupe_minutes=BURST_DEDUPE_MINUTES,
+            )
+    if kind in BURST_KINDS:
+        n = conn.execute(
+            "SELECT count(*), count(DISTINCT account) FROM signin_attempts WHERE NOT ok"
+            " AND reason IS DISTINCT FROM %s AND kind = ANY(%s) AND at > %s",
+            (ratelimit.RATE_LIMITED, list(BURST_KINDS), now - timedelta(minutes=secsettings.BURST_WINDOW_MINUTES)),
+        ).fetchone()
+        failures, accounts = int(n[0]), int(n[1])
+        if failures >= sec.signin_burst_threshold and not _recent_audit(
+            conn, "auth.signin_burst", "burst", BURST_DEDUPE_MINUTES
+        ):
+            detail = {"reason": "burst", "failures": failures, "accounts": accounts}
+            _security_audit(conn, "auth.signin_burst", "burst", detail, ip=ip)
+            notify(
+                conn,
+                SIGNIN_BURST,
+                subject="burst",
+                title=f"{failures} failed sign-ins in {secsettings.BURST_WINDOW_MINUTES} minutes",
+                body=(
+                    f"{failures} failed sign-ins across {accounts} accounts in the last "
+                    f"{secsettings.BURST_WINDOW_MINUTES} minutes (the threshold is {sec.signin_burst_threshold})."
+                ),
+                detail=detail,
+                dedupe_minutes=BURST_DEDUPE_MINUTES,
+            )
+
+
+def _name(conn: Any, user_id: Any) -> str:
+    row = conn.execute("SELECT display_name, sam_account FROM users WHERE id = %s", (user_id,)).fetchone()
+    return f"{row[0]} ({row[1]})" if row else str(user_id)
+
+
+def pin_disabled(conn: Any, user_id: Any, failures: int, where: str) -> None:
+    """A PIN was disabled after too many wrong entries."""
+    name = _name(conn, user_id)
+    notify(
+        conn,
+        SECURITY_ALERT,
+        subject=f"user:{user_id}",
+        title=f"PIN disabled after {failures} wrong entries: {name}",
+        body=f"The PIN of {name} was disabled after {failures} wrong entries ({where}). An Admin can clear it.",
+        detail={"user_id": str(user_id), "failures": failures, "where": where},
+        dedupe_minutes=ALERT_DEDUPE_MINUTES,
+    )
+
+
+def stepup_session_ended(conn: Any, user_id: Any, failures: int, kind: str, ip: str | None) -> None:
+    """A session ended after too many wrong unlock or signature attempts."""
+    name = _name(conn, user_id)
+    what = "unlock" if kind == "unlock" else "signature"
+    notify(
+        conn,
+        SECURITY_ALERT,
+        subject=f"user:{user_id}",
+        title=f"Session ended after {failures} wrong {what} attempts: {name}",
+        body=f"{name} entered a wrong PIN or password {failures} times at {what}. The session was ended. "
+        f"Address: {ip or 'unknown'}.",
+        detail={"user_id": str(user_id), "failures": failures, "kind": kind, "ip": ip},
+        dedupe_minutes=ALERT_DEDUPE_MINUTES,
+    )
+
+
+# ---------------------------------------------------------------- policy denials
+
+_denied_lock = threading.Lock()
+_denied_seen: dict[tuple[str, str, str, str], float] = {}
+
+
+def _denied_due(key: tuple[str, str, str, str]) -> bool:
+    now = time.monotonic()
+    with _denied_lock:
+        last = _denied_seen.get(key)
+        if last is not None and now - last < DENIED_DEDUPE_SECONDS:
+            return False
+        if len(_denied_seen) > 5000:
+            _denied_seen.clear()
+        _denied_seen[key] = now
+        return True
+
+
+def record_denied(settings: Settings, principal: Any, *, method: str, path: str, code: str) -> None:
+    """Audit `policy.denied` for a 403 refused to a signed-in principal. The method, path and code only (never a
+    body or a query string). The same user, path and code is audited once a minute at most."""
+    if code in DENIED_SKIP_CODES:
+        return
+    key = (str(principal.user_id), method, path, code)
+    if not _denied_due(key):
+        return
+    from .. import db
+
+    try:
+        with db.get_pool(settings).connection(timeout=5.0) as conn:
+            conn.autocommit = True
+            audit.append(
+                conn,
+                action="policy.denied",
+                user_id=principal.user_id,
+                session_id=principal.session_id,
+                role_snapshot=principal.role_snapshot,
+                subject_type="route",
+                subject_id=f"{method} {path}"[:300],
+                detail={"method": method, "path": path[:300], "code": code},
+                ip=principal.ip,
+            )
+    except Exception as e:
+        log.warning("could not audit a denial: %s", type(e).__name__)
+
+
+# ---------------------------------------------------------------- the Security tab's list
+
+
+def _row(kind: str, r: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "at": r["at"],
+        "kind": kind,
+        "user_id": str(r["user_id"]) if r.get("user_id") else None,
+        "display_name": r.get("display_name"),
+        "account": r.get("account"),
+        "ip": r.get("ip"),
+        "detail": r.get("detail") or {},
+    }
+
+
+AUDIT_ALERTS = (
+    # (kind, WHERE on audit_events e)
+    ("lockout", "e.action = 'auth.lockout'"),
+    ("signin_burst", "e.action = 'auth.signin_burst'"),
+    ("denied", "e.action = 'policy.denied'"),
+    ("admin_change", "e.action LIKE 'admin.%%'"),
+    ("it_edit", "e.it_reason IS NOT NULL"),
+)
+
+
+AUDIT_ALERT_SQL = """
+SELECT e.at, e.user_id, u.display_name, coalesce(e.detail->>'account', u.sam_account) AS account,
+       host(e.ip) AS ip, e.action, e.detail, e.it_reason, e.subject_type, e.subject_id
+FROM audit_events e LEFT JOIN users u ON u.id = e.user_id WHERE {where} AND e.at > %s
+ORDER BY e.id DESC LIMIT %s
+"""
+
+
+def alerts(conn: Any, hours: int) -> list[dict[str, Any]]:
+    """What happened in the last `hours`, newest first, at most ALERTS_MAX rows."""
+    out: list[dict[str, Any]] = []
+    since = conn.execute("SELECT now() - make_interval(hours => %s::int)", (hours,)).fetchone()[0]
+    with conn.cursor(row_factory=dict_row) as cur:
+        for kind, where in AUDIT_ALERTS:
+            rows = cur.execute(
+                AUDIT_ALERT_SQL.format(where=where),
+                (since, ALERTS_MAX),
+            ).fetchall()
+            for r in rows:
+                detail = dict(r["detail"] or {}) if isinstance(r["detail"], dict) else {"value": r["detail"]}
+                detail.setdefault("action", r["action"])
+                if r["subject_type"]:
+                    detail.setdefault("subject", f"{r['subject_type']}:{r['subject_id']}")
+                if kind == "it_edit":
+                    detail["it_reason"] = r["it_reason"]
+                out.append(_row(kind, {**r, "detail": detail}))
+        rows = cur.execute(
+            "SELECT max(a.at) AS at, a.user_id, u.display_name, u.sam_account AS account, count(*) AS n,"
+            " (array_agg(host(a.ip) ORDER BY a.at DESC))[1] AS ip, array_agg(DISTINCT a.kind) AS kinds"
+            " FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
+            " WHERE NOT a.ok AND a.reason = 'bad_pin' AND a.user_id IS NOT NULL AND a.at > %s"
+            " GROUP BY a.user_id, u.display_name, u.sam_account HAVING count(*) >= %s",
+            (since, PIN_FAILURES_MIN),
+        ).fetchall()
+        for r in rows:
+            out.append(_row("pin_failures", {**r, "detail": {"wrong_pins": int(r["n"]), "where": sorted(r["kinds"])}}))
+        rows = cur.execute(
+            "SELECT a.at, a.user_id, u.display_name, u.sam_account AS account, host(a.ip) AS ip, a.kind, a.reason"
+            " FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
+            " WHERE NOT a.ok AND a.kind IN ('unlock', 'signature') AND a.reason IN ('bad_pin', 'bad_password')"
+            " AND a.at > %s ORDER BY a.id DESC LIMIT %s",
+            (since, ALERTS_MAX),
+        ).fetchall()
+        for r in rows:
+            out.append(_row("stepup_wrong_pin", {**r, "detail": {"where": r["kind"], "reason": r["reason"]}}))
+        rows = cur.execute(
+            "SELECT f.at, f.kind, f.subject, f.detail, f.run_id FROM integrity_findings f WHERE f.at > %s"
+            " ORDER BY f.id DESC LIMIT %s",
+            (since, ALERTS_MAX),
+        ).fetchall()
+        for r in rows:
+            detail = {"finding": r["kind"], "subject": r["subject"], "run_id": r["run_id"], **(r["detail"] or {})}
+            out.append(_row("integrity", {"at": r["at"], "detail": detail}))
+    out.sort(key=lambda x: x["at"], reverse=True)
+    return out[:ALERTS_MAX]