Phase 9: Admin completion, reconcile, notifications, integrity, backups
ci / frontend (pull_request) Successful in 1m11s
ci / server (pull_request) Successful in 12m2s
ci / image (pull_request) Skipped
ci / e2e (pull_request) Successful in 6m1s

Admin: Departments (name and stations; add shows the three groups IT must
create), Profiles (welder stamp, station limits, active) with a CSV import
that previews and applies only when clean, Checklists (weld and plain),
Security (session, lockout and rate-limit settings with bounds, alerts for
the last 24 hours, the integrity result), Access log with filters and a
guarded CSV, Who is in, Stamp check, Notifications, and the employeeID
flags. Every Admin change is audited.

Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh
(15 minutes), integrity (seals, asset hashes, missing and orphan files, the
audit chain), backup-check, notify-send and notify-digest. Notifications
route per event type to AD groups, users, mailboxes or the department
Supervisor group, immediately or in a daily digest, over STARTTLS verified
against the PESCO CA, with dry run on the test stack.

Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer
at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21.
CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19.

Migration 0010. Closes the app part of I6, M3, O1, O2 and S3.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-21 18:27:09 -06:00
co-authored by Claude Opus 5.5
parent ce28d2f3c6
commit a82e056b6a
116 changed files with 15125 additions and 297 deletions
+26 -14
View File
@@ -593,33 +593,45 @@ def open_quota_alert(conn: Any) -> dict[str, Any] | None:
def check_quota(conn: Any, settings: Settings) -> dict[str, Any]:
"""Open or resolve the storage alert. Returns the usage with `alert`: "opened", "resolved" or None.
Over the threshold with no open `assets_quota` row: insert one (severity warning, subject assets, detail the
usage) and audit `storage.quota_warning`. Under it with one open: set its resolved_at.
Over the threshold with no open `assets_quota` event: raise one through notify.raise_event (a standing
condition, subject assets, detail the usage) and audit `storage.quota_warning`. Under it with one open:
notify.resolve_event closes it and queues the recovery notice.
"""
from ..notify import raise_event, resolve_event
usage = usage_for(conn, settings)
change: str | None = None
warn_pct = int(settings.assets_quota_warn_pct)
with conn.transaction():
open_row = open_quota_alert(conn)
if usage["warn"] and open_row is None:
r = conn.execute(
"INSERT INTO notification_events (kind, severity, subject, detail) VALUES (%s, 'warning', %s, %s)"
" ON CONFLICT (kind, coalesce(subject, '')) WHERE resolved_at IS NULL DO NOTHING RETURNING id",
(QUOTA_KIND, QUOTA_SUBJECT, Jsonb(usage)),
).fetchone()
if r is not None:
new_id = raise_event(
conn,
QUOTA_KIND,
subject=QUOTA_SUBJECT,
title=f"Assets storage is {usage['pct']}% full",
body=f"The assets storage is {usage['pct']}% full. The warning is at {warn_pct}%."
f" Used {usage['used']} of {usage['total']} bytes.",
detail=usage,
condition=True,
)
if new_id is not None:
change = "opened"
audit.append(
conn,
action="storage.quota_warning",
subject_type="notification",
subject_id=str(r[0]),
detail={**usage, "warn_pct": int(settings.assets_quota_warn_pct)},
)
log.warning(
"assets storage is %.1f%% full (warning at %s%%)", usage["pct"], settings.assets_quota_warn_pct
subject_id=str(new_id),
detail={**usage, "warn_pct": warn_pct},
)
log.warning("assets storage is %.1f%% full (warning at %s%%)", usage["pct"], warn_pct)
elif not usage["warn"] and open_row is not None:
conn.execute("UPDATE notification_events SET resolved_at = now() WHERE id = %s", (open_row["id"],))
resolve_event(
conn,
QUOTA_KIND,
subject=QUOTA_SUBJECT,
body=f"The assets storage is back under the warning: {usage['pct']}% full.",
)
change = "resolved"
log.info("assets storage is back under the warning: %.1f%%", usage["pct"])
return {**usage, "alert": change}
+266
View File
@@ -0,0 +1,266 @@
"""Departments and their stations, as the Admin edits them.
Code and slug never change (the group names derive from them; a trigger refuses it too). Stations are never deleted,
because station limits and packets point at them: `active: false` hides one. A new department is a shop department
with the three shop groups IT must create in the PacketStation OU. The support departments map to fixed roles, so a
new one would have no group and no role; it is refused.
"""
from __future__ import annotations
import re
from typing import Any
from psycopg.rows import dict_row
from .. import audit
from ..api.errors import ApiError
from ..auth import groups as groupmap
RX_CODE = re.compile(r"^[0-9A-Z]{2,4}$")
RX_SLUG = re.compile(r"^[A-Za-z][A-Za-z0-9]{1,30}$")
NAME_MAX = 80
STATION_NAME_MAX = 60
STATIONS_MAX = 60
def clean_name(value: Any, what: str, limit: int) -> str:
text = " ".join(str(value or "").split())
if not text:
raise ApiError(422, "invalid_request", f"Give the {what} a name.")
if len(text) > limit:
raise ApiError(422, "invalid_request", f"The {what} name is longer than {limit} characters.")
return text
def group_names_for(code: str, slug: str, kind: str, supervisor_group: bool) -> list[tuple[str, str]]:
"""(group name, role) for one department, as the group map builds them."""
if kind == "shop":
return [(f"PS-{code}-{slug}-{lv.capitalize()}", f"shop:{code}:{lv}") for lv in groupmap.SHOP_LEVELS]
kinds = groupmap.OFFICE_KINDS.get(code)
if kinds is None:
return []
member, sup = kinds
out = [(f"PS-{slug}", member)]
if supervisor_group and sup:
out.append((f"PS-{slug}-Supervisor", sup))
return out
def _stations(conn: Any, code: str | None = None) -> dict[str, list[dict[str, Any]]]:
sql = "SELECT dept_code, id, name, sort, active FROM stations"
params: tuple[Any, ...] = ()
if code is not None:
sql += " WHERE dept_code = %s"
params = (code,)
out: dict[str, list[dict[str, Any]]] = {}
for dept, sid, name, sort, active in conn.execute(sql + " ORDER BY dept_code, sort, name, id", params):
out.setdefault(dept, []).append({"id": int(sid), "name": name, "sort": int(sort), "active": bool(active)})
return out
def _out(r: dict[str, Any], stations: list[dict[str, Any]], in_ou: set[str] | None) -> dict[str, Any]:
groups = [
{"name": name, "role": role, "in_ou": None if in_ou is None else name.lower() in in_ou}
for name, role in group_names_for(r["code"], r["slug"], r["kind"], bool(r["supervisor_group"]))
]
return {
"code": r["code"],
"slug": r["slug"],
"name": r["name"],
"kind": r["kind"],
"sort": int(r["sort"]),
"active": bool(r["active"]),
"weld": bool(r["weld"]),
"supervisor_group": bool(r["supervisor_group"]),
"created_at": r["created_at"],
"groups": groups,
"stations": stations,
}
SELECT = (
"SELECT code, slug, name, kind::text AS kind, sort, active, weld, supervisor_group, created_at FROM departments"
)
def list_all(conn: Any, in_ou: list[str] | None) -> list[dict[str, Any]]:
"""Every department, active or not, in sort order. `in_ou` is the group names in the OU (None: unknown)."""
lower = {n.lower() for n in in_ou} if in_ou is not None else None
with conn.cursor(row_factory=dict_row) as cur:
rows = cur.execute(SELECT + " ORDER BY sort, code").fetchall()
st = _stations(conn)
return [_out(r, st.get(r["code"], []), lower) for r in rows]
def get(conn: Any, code: str, in_ou: list[str] | None = None) -> dict[str, Any] | None:
lower = {n.lower() for n in in_ou} if in_ou is not None else None
with conn.cursor(row_factory=dict_row) as cur:
r = cur.execute(SELECT + " WHERE code = %s", (code,)).fetchone()
if r is None:
return None
return _out(r, _stations(conn, code).get(code, []), lower)
def _snapshot(d: dict[str, Any]) -> dict[str, Any]:
return {k: d[k] for k in ("name", "active", "stations")}
def update(
conn: Any,
who: Any,
code: str,
*,
name: str | None,
active: bool | None,
stations: list[dict[str, Any]] | None,
) -> dict[str, Any]:
"""Change the name, active and stations of one department. Audited `admin.department_updated`."""
with conn.transaction():
locked = conn.execute("SELECT code FROM departments WHERE code = %s FOR UPDATE", (code,)).fetchone()
if locked is None:
raise ApiError(404, "not_found", "No such department.")
before = get(conn, code)
assert before is not None
if name is not None:
conn.execute(
"UPDATE departments SET name = %s WHERE code = %s", (clean_name(name, "department", NAME_MAX), code)
)
if active is not None:
conn.execute("UPDATE departments SET active = %s WHERE code = %s", (bool(active), code))
if stations is not None:
_apply_stations(conn, code, before["stations"], stations)
after = get(conn, code)
assert after is not None
if _snapshot(before) != _snapshot(after):
audit.append(
conn,
action="admin.department_updated",
user_id=who.user_id,
session_id=who.session_id,
role_snapshot=who.role_snapshot,
subject_type="department",
subject_id=code,
detail={"before": _snapshot(before), "after": _snapshot(after)},
ip=who.ip,
)
return after
def _apply_stations(conn: Any, code: str, current: list[dict[str, Any]], wanted: list[dict[str, Any]]) -> None:
by_id = {s["id"]: s for s in current}
final: dict[int, dict[str, Any]] = {s["id"]: dict(s) for s in current}
new: list[dict[str, Any]] = []
seen_ids: set[int] = set()
for w in wanted:
sid = w.get("id")
entry = {
"name": clean_name(w.get("name"), "station", STATION_NAME_MAX),
"sort": int(w.get("sort") or 0),
"active": bool(w.get("active", True)),
}
if sid is None:
new.append(entry)
continue
if sid not in by_id:
raise ApiError(422, "invalid_request", f"Station {sid} is not in department {code}.")
if sid in seen_ids:
raise ApiError(422, "invalid_request", f"Station {sid} is listed twice.")
seen_ids.add(sid)
final[sid].update(entry)
names = [s["name"].lower() for s in final.values()] + [s["name"].lower() for s in new]
dupes = sorted({n for n in names if names.count(n) > 1})
if dupes:
raise ApiError(422, "duplicate_station", f"Two stations in {code} have the same name: {', '.join(dupes)}.")
if len(final) + len(new) > STATIONS_MAX:
raise ApiError(422, "invalid_request", f"A department has at most {STATIONS_MAX} stations.")
renamed = [sid for sid, s in final.items() if s["name"] != by_id[sid]["name"]]
# Names are unique per department: park the renamed ones first so two stations can swap names.
for sid in renamed:
conn.execute("UPDATE stations SET name = %s WHERE id = %s", (f"~{sid}~renaming", sid))
for sid, s in final.items():
if s != by_id[sid]:
conn.execute(
"UPDATE stations SET name = %s, sort = %s, active = %s WHERE id = %s",
(s["name"], s["sort"], s["active"], sid),
)
for s in new:
conn.execute(
"INSERT INTO stations (dept_code, name, sort, active) VALUES (%s, %s, %s, %s)",
(code, s["name"], s["sort"], s["active"]),
)
def add(
conn: Any,
who: Any,
*,
code: str,
slug: str,
name: str,
kind: str,
weld: bool,
supervisor_group: bool,
stations: list[str],
) -> tuple[dict[str, Any], list[str]]:
"""Add a shop department. Returns (department, group names IT must create). Audited `admin.department_added`."""
code, slug = code.strip(), slug.strip()
if not RX_CODE.match(code):
raise ApiError(422, "invalid_request", "The code is 2 to 4 capital letters or digits, like 15.")
if not RX_SLUG.match(slug):
raise ApiError(422, "invalid_request", "The short name is a letter, then letters or digits, no spaces.")
if kind != "shop":
raise ApiError(
422,
"support_fixed",
"Only a shop department can be added. The support departments map to fixed roles.",
)
title = clean_name(name, "department", NAME_MAX)
names: list[str] = []
for s in stations:
n = clean_name(s, "station", STATION_NAME_MAX)
if n.lower() in (x.lower() for x in names):
raise ApiError(422, "duplicate_station", f"Two stations have the same name: {n}.")
names.append(n)
if len(names) > STATIONS_MAX:
raise ApiError(422, "invalid_request", f"A department has at most {STATIONS_MAX} stations.")
with conn.transaction():
conn.execute("LOCK TABLE departments IN SHARE ROW EXCLUSIVE MODE")
if conn.execute("SELECT 1 FROM departments WHERE upper(code) = upper(%s)", (code,)).fetchone():
raise ApiError(409, "code_taken", f"Department code {code} is taken.")
if conn.execute("SELECT 1 FROM departments WHERE lower(slug) = lower(%s)", (slug,)).fetchone():
raise ApiError(409, "slug_taken", f"The short name {slug} is taken.")
sort = int(conn.execute("SELECT coalesce(max(sort), 0) + 10 FROM departments").fetchone()[0])
conn.execute(
"INSERT INTO departments (code, slug, name, kind, sort, active, supervisor_group, weld, created_at,"
" created_by) VALUES (%s, %s, %s, %s::dept_kind, %s, true, %s, %s, now(), %s)",
(code, slug, title, kind, sort, bool(supervisor_group), bool(weld), who.user_id),
)
for i, n in enumerate(names, start=1):
conn.execute("INSERT INTO stations (dept_code, name, sort) VALUES (%s, %s, %s)", (code, n, i))
dept = get(conn, code)
assert dept is not None
groups = [g["name"] for g in dept["groups"]]
audit.append(
conn,
action="admin.department_added",
user_id=who.user_id,
session_id=who.session_id,
role_snapshot=who.role_snapshot,
subject_type="department",
subject_id=code,
detail={
"after": {
"code": code,
"slug": slug,
"name": title,
"kind": kind,
"weld": bool(weld),
"supervisor_group": bool(supervisor_group),
"stations": names,
},
"groups_to_create": groups,
},
ip=who.ip,
)
return dept, groups
+34
View File
@@ -30,6 +30,7 @@ from ..api import deps
from ..api.errors import ApiError
from ..auth import signatures
from ..auth.signatures import Signature
from ..notify import raise_event
from ..policy.actions import SCOPED_LABELS
from ..policy.engine import check_scoped
from ..text import LIMITS, clean_opt, reason_ok
@@ -593,5 +594,38 @@ def _finish(w: ItpWrite) -> None:
events.publish(conn, {"event": "changed", "kind": "itp", "unit_id": w.itp["unit_id"], "itp_id": w.id})
def reject_dept(conn: Any, unit_id: Any, page: Any) -> str | None:
"""The department of a rejected line: the one department among the unit's live packets that works to the
line's page, or None when there is none or more than one."""
codes = {
str(r[0])
for r in conn.execute(
"SELECT DISTINCT dept_code FROM packets WHERE unit_id = %s AND removed_at IS NULL", (unit_id,)
).fetchall()
}
hits = sorted(c for c in codes if page and page in DEPT_PAGES.get(c, ()))
return hits[0] if len(hits) == 1 else None
def notify_reject(w: ItpWrite, key: str, ncr: str | None, note: str | None) -> int | None:
"""qc_reject for an ITP line reject, in the write's transaction (notify-send mails it)."""
item = next((i for i in items(w.itp.get("template") or {}) if str(i.get("n")) == key), {})
page = item.get("page")
dept = reject_dept(w.conn, w.itp["unit_id"], page)
who = (w.principal.user or {}).get("display_name") or ""
head = f"NCR {ncr}" if ncr else "NCR # pending"
lines = [str(item.get("t") or ""), head, note or "", f"Rejected by {who}." if who else ""]
where = f" ({PAGE_NAMES.get(str(page), page)})" if page else ""
return raise_event(
w.conn,
"qc_reject",
subject=f"itp:{w.id}:{key}",
title=f"ITP {w.itp.get('no') or w.id} line {key} rejected{where}",
body="\n".join(x for x in lines if x),
detail={"itp_id": w.id, "unit_id": w.itp["unit_id"], "line": key, "page": page, "ncr": ncr},
dept_code=dept,
)
def label(action: policy.Action) -> str:
return SCOPED_LABELS.get(action, str(action))
+370
View File
@@ -0,0 +1,370 @@
"""People's Packet Station profiles: welder stamp, profile active, station limits. Edited one at a time or by a CSV
import keyed on the employee ID.
Roles never come from here; AD groups decide them. An inactive profile drops out of the welder picker and the
worker filter. Employee IDs stay text (leading zeros kept).
"""
from __future__ import annotations
import csv
import io
import re
from collections import Counter
from collections.abc import Mapping
from typing import Any
from .. import audit
from ..api.errors import ApiError
RX_STAMP = re.compile(r"^[A-Za-z0-9-]{1,12}$")
CSV_MAX_BYTES = 1_000_000
CSV_MAX_ROWS = 2000
COLUMNS = ("employee_id", "welder_stamp", "active", "stations")
CLEAR = "-"
TRUE_WORDS = ("yes", "true", "1")
FALSE_WORDS = ("no", "false", "0")
# ---------------------------------------------------------------- reading
def station_limits(conn: Any, user_ids: list[Any]) -> dict[Any, dict[str, list[int]]]:
"""user id -> {dept_code: [station ids]}."""
out: dict[Any, dict[str, list[int]]] = {}
if not user_ids:
return out
rows = conn.execute(
"SELECT l.user_id, s.dept_code, s.id FROM user_station_limits l JOIN stations s ON s.id = l.station_id"
" WHERE l.user_id = ANY(%s) ORDER BY s.dept_code, s.sort, s.id",
(user_ids,),
).fetchall()
for uid, dept, sid in rows:
out.setdefault(uid, {}).setdefault(dept, []).append(int(sid))
return out
def profile_of(conn: Any, user_id: Any) -> dict[str, Any] | None:
row = conn.execute("SELECT welder_stamp, profile_active FROM users WHERE id = %s", (user_id,)).fetchone()
if row is None:
return None
return {
"welder_stamp": row[0],
"profile_active": bool(row[1]),
"station_limits": station_limits(conn, [user_id]).get(user_id, {}),
}
def _stamp_owner(conn: Any, stamp: str, user_id: Any) -> str | None:
row = conn.execute(
"SELECT display_name FROM users WHERE lower(welder_stamp) = lower(%s) AND id <> %s LIMIT 1", (stamp, user_id)
).fetchone()
return str(row[0]) if row else None
# ---------------------------------------------------------------- one person
def _audit(conn: Any, who: Any, user_id: Any, before: dict[str, Any], after: dict[str, Any], source: str) -> None:
audit.append(
conn,
action="admin.profile_updated",
user_id=who.user_id,
session_id=who.session_id,
role_snapshot=who.role_snapshot,
subject_type="user",
subject_id=user_id,
detail={"before": before, "after": after, "source": source},
ip=who.ip,
)
def _write(conn: Any, user_id: Any, stamp: Any, active: Any, limits: Mapping[str, list[int]] | None) -> None:
"""Write the fields given; `stamp` and `active` use ... (Ellipsis) for unchanged. `limits` replaces the named
departments' limits; None leaves them."""
if stamp is not ...:
conn.execute("UPDATE users SET welder_stamp = %s WHERE id = %s", (stamp, user_id))
if active is not ...:
conn.execute("UPDATE users SET profile_active = %s WHERE id = %s", (bool(active), user_id))
for dept, ids in (limits or {}).items():
conn.execute(
"DELETE FROM user_station_limits l USING stations s WHERE s.id = l.station_id AND l.user_id = %s"
" AND s.dept_code = %s",
(user_id, dept),
)
for sid in ids:
conn.execute("INSERT INTO user_station_limits (user_id, station_id) VALUES (%s, %s)", (user_id, sid))
def update(conn: Any, who: Any, user_id: Any, fields: Mapping[str, Any]) -> dict[str, Any]:
"""PATCH a profile: welder_stamp (None clears), profile_active, station_limits {dept: [station id]} (a dept key
replaces that department's limits; an empty list clears them)."""
stamp: Any = ...
if "welder_stamp" in fields:
raw = fields["welder_stamp"]
stamp = None if raw is None or not str(raw).strip() else str(raw).strip()
if stamp is not None and not RX_STAMP.match(stamp):
raise ApiError(422, "bad_stamp", "A stamp is 1 to 12 letters, digits or dashes.")
active: Any = fields["profile_active"] if fields.get("profile_active") is not None else ...
limits: dict[str, list[int]] | None = None
if fields.get("station_limits") is not None:
limits = {}
for dept, ids in dict(fields["station_limits"]).items():
ok = {int(r[0]) for r in conn.execute("SELECT id FROM stations WHERE dept_code = %s", (dept,)).fetchall()}
if not conn.execute("SELECT 1 FROM departments WHERE code = %s", (dept,)).fetchone():
raise ApiError(422, "unknown_department", f"No department {dept}.")
want = sorted({int(x) for x in ids})
bad = [x for x in want if x not in ok]
if bad:
raise ApiError(422, "unknown_station", f"Station {bad[0]} is not in department {dept}.")
limits[str(dept)] = want
with conn.transaction():
if conn.execute("SELECT 1 FROM users WHERE id = %s FOR UPDATE", (user_id,)).fetchone() is None:
raise ApiError(404, "not_found", "No such user.")
if stamp is not ... and stamp is not None:
owner = _stamp_owner(conn, stamp, user_id)
if owner is not None:
raise ApiError(409, "stamp_taken", f"Stamp {stamp} belongs to {owner}.")
before = profile_of(conn, user_id)
assert before is not None
_write(conn, user_id, stamp, active, limits)
after = profile_of(conn, user_id)
assert after is not None
if before != after:
_audit(conn, who, user_id, before, after, "edit")
return after
# ---------------------------------------------------------------- CSV import
def _station_index(conn: Any) -> tuple[set[str], dict[tuple[str, str], int], dict[int, str]]:
depts = {r[0] for r in conn.execute("SELECT code FROM departments").fetchall()}
by_name: dict[tuple[str, str], int] = {}
label: dict[int, str] = {}
for sid, dept, name in conn.execute("SELECT id, dept_code, name FROM stations").fetchall():
by_name[(dept.lower(), " ".join(name.split()).lower())] = int(sid)
label[int(sid)] = f"{dept}:{name}"
return depts, by_name, label
def _bool(text: str) -> bool | None:
t = text.strip().lower()
if t in TRUE_WORDS:
return True
if t in FALSE_WORDS:
return False
return None
def _labels(ids: list[int], label: dict[int, str]) -> list[str]:
return sorted(label.get(i, str(i)) for i in ids)
def parse_csv(text: str) -> tuple[list[str], list[tuple[int, dict[str, str]]]]:
"""(header columns, [(line, {column: cell})]). Raises ApiError for a file that cannot be read at all."""
if len(text.encode("utf-8")) > CSV_MAX_BYTES:
raise ApiError(413, "too_large", "The file is larger than 1 MB.")
reader = csv.reader(io.StringIO(text.lstrip("")))
try:
header = next(reader, None)
if header is None:
raise ApiError(422, "bad_csv", "The file is empty.")
cols = [h.strip().lower() for h in header]
unknown = [c for c in cols if c and c not in COLUMNS]
if "employee_id" not in cols:
raise ApiError(422, "bad_csv", "The header row has no employee_id column.")
if unknown:
raise ApiError(422, "bad_csv", f"Unknown column: {', '.join(unknown)}.")
if len(set(c for c in cols if c)) != len([c for c in cols if c]):
raise ApiError(422, "bad_csv", "A column is named twice.")
rows: list[tuple[int, dict[str, str]]] = []
for cells in reader:
if not any(c.strip() for c in cells):
continue
if len(rows) >= CSV_MAX_ROWS:
raise ApiError(422, "bad_csv", f"The file has more than {CSV_MAX_ROWS} rows.")
row = {c: (cells[i].strip() if i < len(cells) else "") for i, c in enumerate(cols) if c}
rows.append((reader.line_num, row))
except csv.Error as e:
raise ApiError(422, "bad_csv", f"The file is not valid CSV: {e}.") from None
return cols, rows
def import_profiles(conn: Any, who: Any, text: str, *, apply: bool) -> dict[str, Any]:
"""Preview (apply false) or apply a profile CSV. With any error, apply is refused: 422 with the same body."""
_, parsed = parse_csv(text)
depts, by_name, label = _station_index(conn)
emps = [r.get("employee_id", "") for _, r in parsed]
users: dict[str, list[dict[str, Any]]] = {}
want = sorted({e for e in emps if e})
if want:
for uid, emp, name, stamp, active in conn.execute(
"SELECT id, emp_no, display_name, welder_stamp, profile_active FROM users"
" WHERE active AND emp_no = ANY(%s)",
(want,),
).fetchall():
users.setdefault(emp, []).append(
{"id": uid, "display_name": name, "welder_stamp": stamp, "profile_active": bool(active)}
)
limits = station_limits(conn, [u["id"] for us in users.values() for u in us])
seen: Counter[str] = Counter()
first_line: dict[str, int] = {}
out: list[dict[str, Any]] = []
plans: list[tuple[dict[str, Any], dict[str, Any]]] = [] # (row out, plan)
for line, r in parsed:
emp = r.get("employee_id", "")
errors: list[str] = []
row_out: dict[str, Any] = {
"line": line,
"employee_id": emp,
"user_id": None,
"display_name": None,
"changes": {},
"errors": errors,
}
out.append(row_out)
if not emp:
errors.append("Employee ID is missing.")
continue
seen[emp] += 1
if seen[emp] > 1:
errors.append(f"Employee ID {emp} is repeated in the file (first on line {first_line[emp]}).")
continue
first_line[emp] = line
found = users.get(emp, [])
if not found:
errors.append(f"Unknown employee ID {emp}.")
continue
if len(found) > 1:
errors.append(f"Employee ID {emp} is on more than one user.")
continue
u = found[0]
row_out["user_id"] = str(u["id"])
row_out["display_name"] = u["display_name"]
plan: dict[str, Any] = {"user": u}
stamp_cell = r.get("welder_stamp", "")
if stamp_cell:
new_stamp = None if stamp_cell == CLEAR else stamp_cell
if new_stamp is not None and not RX_STAMP.match(new_stamp):
errors.append(f"Bad stamp {stamp_cell}: 1 to 12 letters, digits or dashes.")
elif new_stamp != u["welder_stamp"]:
plan["welder_stamp"] = new_stamp
active_cell = r.get("active", "")
if active_cell:
val = _bool(active_cell)
if val is None:
errors.append(f"Bad active value {active_cell}: use yes or no.")
elif val != u["profile_active"]:
plan["profile_active"] = val
st_cell = r.get("stations", "")
if st_cell:
current = sorted(i for ids in limits.get(u["id"], {}).values() for i in ids)
ids: list[int] = []
if st_cell != CLEAR:
for part in (p.strip() for p in st_cell.split(";")):
if not part:
continue
dept, sep, name = part.partition(":")
dept, name = dept.strip(), " ".join(name.split())
if not sep or not dept or not name:
errors.append(f"Bad station {part}: write <department code>:<station name>.")
elif dept not in depts:
errors.append(f"Unknown department {dept}.")
elif (dept.lower(), name.lower()) not in by_name:
errors.append(f"Unknown station {name} in department {dept}.")
else:
sid = by_name[(dept.lower(), name.lower())]
if sid not in ids:
ids.append(sid)
ids.sort()
if ids != current:
plan["stations"] = (current, ids)
plans.append((row_out, plan))
# Stamps must stay unique after the whole file applies.
final: dict[Any, str | None] = {}
for _, plan in plans:
if "welder_stamp" in plan:
final[plan["user"]["id"]] = plan["welder_stamp"]
claimed = [s for s in final.values() if s]
if claimed:
others = {
str(r[1]).lower(): (r[0], r[2])
for r in conn.execute(
"SELECT id, welder_stamp, display_name FROM users WHERE welder_stamp IS NOT NULL"
" AND lower(welder_stamp) = ANY(%s)",
([s.lower() for s in claimed],),
).fetchall()
}
names_by_id = {plan["user"]["id"]: plan["user"]["display_name"] for _, plan in plans}
taken_in_file: dict[str, Any] = {}
for row_out, plan in plans:
s = plan.get("welder_stamp")
if not s:
continue
uid = plan["user"]["id"]
holder = others.get(s.lower())
# The holder keeps the stamp unless the same file gives them another (or clears it)
freed = holder is not None and holder[0] in final and str(final[holder[0]] or "").lower() != s.lower()
if holder is not None and holder[0] != uid and not freed:
row_out["errors"].append(f"Stamp {s} is taken by {holder[1]}.")
elif s.lower() in taken_in_file:
row_out["errors"].append(f"Stamp {s} is taken by {names_by_id[taken_in_file[s.lower()]]}.")
else:
taken_in_file[s.lower()] = uid
for row_out, plan in plans:
u = plan["user"]
if "welder_stamp" in plan:
row_out["changes"]["welder_stamp"] = {"from": u["welder_stamp"], "to": plan["welder_stamp"]}
if "profile_active" in plan:
row_out["changes"]["profile_active"] = {"from": u["profile_active"], "to": plan["profile_active"]}
if "stations" in plan:
cur, ids = plan["stations"]
row_out["changes"]["station_limits"] = {"from": _labels(cur, label), "to": _labels(ids, label)}
n_err = sum(1 for r in out if r["errors"])
body: dict[str, Any] = {"rows": out, "ok": len(out) - n_err, "errors": n_err, "applied": False}
if not apply:
return body
if n_err:
raise ApiError(
422,
"import_errors",
f"{n_err} row{'s' if n_err != 1 else ''} with errors. Nothing was changed.",
extra=body,
)
changed = 0
with conn.transaction():
for row_out, plan in plans:
if not row_out["changes"]:
continue
uid = plan["user"]["id"]
conn.execute("SELECT 1 FROM users WHERE id = %s FOR UPDATE", (uid,))
before = profile_of(conn, uid)
stamp = plan.get("welder_stamp", ...)
active = plan.get("profile_active", ...)
lim: dict[str, list[int]] | None = None
if "stations" in plan:
conn.execute("DELETE FROM user_station_limits WHERE user_id = %s", (uid,))
for sid in plan["stations"][1]:
conn.execute("INSERT INTO user_station_limits (user_id, station_id) VALUES (%s, %s)", (uid, sid))
_write(conn, uid, stamp, active, lim)
after = profile_of(conn, uid)
assert before is not None and after is not None
_audit(conn, who, uid, before, after, "import")
changed += 1
audit.append(
conn,
action="admin.profile_import",
user_id=who.user_id,
session_id=who.session_id,
role_snapshot=who.role_snapshot,
subject_type="users",
subject_id="import",
detail={"rows": len(out), "changed": changed, "unchanged": len(out) - changed},
ip=who.ip,
)
body["applied"] = True
return body
+19
View File
@@ -10,6 +10,7 @@ from collections.abc import Mapping
from typing import Any
from .. import policy
from ..notify import raise_event
from ..text import LIMITS
from . import itp, packets, qcdocs, search
@@ -65,6 +66,23 @@ def inspector_text(insp: Mapping[str, Any] | None) -> str:
return f"{who} [{stamp}]" if stamp else who
def _notify_reject(w: Any, why: str) -> None:
"""qc_reject for the packet's department, in the write's transaction (notify-send mails it)."""
pkt = w.packet
dept = str(w.from_dept or pkt.get("dept_code") or "") or None
what = str(pkt.get("title") or pkt.get("drawing") or "")
who = (w.principal.user or {}).get("display_name") or ""
raise_event(
w.conn,
"qc_reject",
subject=f"packet:{w.id}",
title=f"Q.C. rejected packet {w.id}" + (f" ({what})" if what else "") + (f", dept {dept}" if dept else ""),
body="\n".join(x for x in (f"Rejected by {who}." if who else "", why) if x),
detail={"packet_id": w.id, "dept_code": dept, "by": str(w.user_id)},
dept_code=dept,
)
def qc_texts(w: Any, a: policy.Action, to_dept: str) -> None:
"""The packet_log label and note of a Q.C. accept or reject. Called before the side effects close the inspection.
@@ -79,6 +97,7 @@ def qc_texts(w: Any, a: policy.Action, to_dept: str) -> None:
if welds:
parts.append("welds " + ", ".join(welds))
w.log("Q.C. rejected → supervisor", " · ".join(x for x in parts if x) or None)
_notify_reject(w, " · ".join(x for x in parts if x))
return
if a is A.QC_ACCEPT_AND_ROUTE:
labels = getattr(w.children, "dept_labels", None) or packets.dept_labels(w.conn)
+2 -1
View File
@@ -369,7 +369,8 @@ JOIN stations st ON st.id = p.station_id WHERE p.removed_at IS NULL{scope} ORDER
"""
FACET_WORKERS_SQL = """
SELECT DISTINCT wu.id, wu.display_name FROM packets p JOIN units u ON u.id = p.unit_id
JOIN users wu ON wu.id = p.welder_id WHERE p.removed_at IS NULL{scope} ORDER BY wu.display_name, wu.id
JOIN users wu ON wu.id = p.welder_id AND wu.profile_active
WHERE p.removed_at IS NULL{scope} ORDER BY wu.display_name, wu.id
"""
@@ -0,0 +1,321 @@
"""Security alerts: raising them on a failed sign-in, a disabled PIN or an ended session, auditing policy denials,
and the Admin Security tab's list of what happened.
Raising goes through `notify.raise_event` (security_alert, signin_burst) inside the caller's database work. A
failure here is logged and never breaks the sign-in or the request it rides on.
"""
from __future__ import annotations
import logging
import threading
import time
from datetime import timedelta
from typing import Any
from psycopg.rows import dict_row
from .. import audit
from ..auth import ratelimit, secsettings
from ..settings import Settings
log = logging.getLogger(__name__)
SECURITY_ALERT = "security_alert"
SIGNIN_BURST = "signin_burst"
ALERT_DEDUPE_MINUTES = 15
BURST_DEDUPE_MINUTES = 30
BURST_KINDS = ("password", "pin")
PIN_FAILURES_MIN = 3
ALERTS_MAX = 1000
DENIED_DEDUPE_SECONDS = 60.0
DENIED_SKIP_CODES = frozenset({"wrong_secret"})
ALERT_KINDS = (
"lockout",
"pin_failures",
"denied",
"stepup_wrong_pin",
"admin_change",
"it_edit",
"integrity",
"signin_burst",
)
def notify(conn: Any, event_type: str, **kw: Any) -> int | None:
"""notify.raise_event in a savepoint. A failure is logged and returns None."""
try:
from ..notify import raise_event
with conn.transaction():
result: int | None = raise_event(conn, event_type, **kw)
return result
except Exception:
log.exception("could not raise %s", event_type)
return None
def _recent_audit(conn: Any, action: str, subject: str, minutes: int) -> bool:
row = conn.execute(
"SELECT 1 FROM audit_events WHERE action = %s AND subject_type = 'security' AND subject_id = %s"
" AND at > now() - make_interval(mins => %s::int) LIMIT 1",
(action, subject, max(1, minutes)),
).fetchone()
return row is not None
def _security_audit(
conn: Any, action: str, subject: str, detail: dict[str, Any], *, user_id: Any = None, ip: str | None = None
) -> None:
audit.append(
conn, action=action, user_id=user_id, subject_type="security", subject_id=subject, detail=detail, ip=ip
)
def after_failure(
conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any = None
) -> None:
"""Call after a failed attempt was recorded. Raises security_alert when the account's cooldown starts, and
signin_burst when the address limit trips or the failures across accounts reach the burst threshold."""
try:
with conn.transaction():
_after_failure(conn, settings, account=account, ip=ip, kind=kind, user_id=user_id)
except Exception:
log.exception("security alert check failed")
def _after_failure(conn: Any, settings: Settings, *, account: str, ip: str | None, kind: str, user_id: Any) -> None:
sec = secsettings.effective(conn, settings)
s = sec.settings or settings
now = ratelimit.db_now(conn)
if account:
wait = ratelimit.check(conn, s, accounts=[account], ip=None, now=now)
subject = f"account:{account}"
if wait is not None and not _recent_audit(conn, "auth.lockout", subject, sec.signin_account_cooldown_minutes):
minutes = max(1, (wait + 59) // 60)
_security_audit(
conn,
"auth.lockout",
subject,
{"account": account, "kind": kind, "minutes": minutes},
user_id=user_id,
ip=ip,
)
notify(
conn,
SECURITY_ALERT,
subject=subject,
title=f"Sign-in cooldown started for {account}",
body=(
f"{sec.signin_account_limit} failed attempts on {account} within "
f"{sec.signin_account_window_minutes} minutes. Further attempts are refused for {minutes} "
f"minutes. Last address: {ip or 'unknown'}."
),
detail={"account": account, "kind": kind, "ip": ip, "minutes": minutes},
dedupe_minutes=ALERT_DEDUPE_MINUTES,
)
if ip:
wait = ratelimit.check(conn, s, accounts=[], ip=ip, now=now)
subject = f"ip:{ip}"
if wait is not None and not _recent_audit(conn, "auth.signin_burst", subject, BURST_DEDUPE_MINUTES):
_security_audit(conn, "auth.signin_burst", subject, {"reason": "ip_limit", "ip": ip}, ip=ip)
notify(
conn,
SIGNIN_BURST,
subject=subject,
title=f"Sign-in limit reached for address {ip}",
body=(
f"{sec.signin_ip_limit} failed sign-ins from {ip} within {sec.signin_ip_window_minutes} minutes. "
"The address is refused until the count drops."
),
detail={"reason": "ip_limit", "ip": ip},
dedupe_minutes=BURST_DEDUPE_MINUTES,
)
if kind in BURST_KINDS:
n = conn.execute(
"SELECT count(*), count(DISTINCT account) FROM signin_attempts WHERE NOT ok"
" AND reason IS DISTINCT FROM %s AND kind = ANY(%s) AND at > %s",
(ratelimit.RATE_LIMITED, list(BURST_KINDS), now - timedelta(minutes=secsettings.BURST_WINDOW_MINUTES)),
).fetchone()
failures, accounts = int(n[0]), int(n[1])
if failures >= sec.signin_burst_threshold and not _recent_audit(
conn, "auth.signin_burst", "burst", BURST_DEDUPE_MINUTES
):
detail = {"reason": "burst", "failures": failures, "accounts": accounts}
_security_audit(conn, "auth.signin_burst", "burst", detail, ip=ip)
notify(
conn,
SIGNIN_BURST,
subject="burst",
title=f"{failures} failed sign-ins in {secsettings.BURST_WINDOW_MINUTES} minutes",
body=(
f"{failures} failed sign-ins across {accounts} accounts in the last "
f"{secsettings.BURST_WINDOW_MINUTES} minutes (the threshold is {sec.signin_burst_threshold})."
),
detail=detail,
dedupe_minutes=BURST_DEDUPE_MINUTES,
)
def _name(conn: Any, user_id: Any) -> str:
row = conn.execute("SELECT display_name, sam_account FROM users WHERE id = %s", (user_id,)).fetchone()
return f"{row[0]} ({row[1]})" if row else str(user_id)
def pin_disabled(conn: Any, user_id: Any, failures: int, where: str) -> None:
"""A PIN was disabled after too many wrong entries."""
name = _name(conn, user_id)
notify(
conn,
SECURITY_ALERT,
subject=f"user:{user_id}",
title=f"PIN disabled after {failures} wrong entries: {name}",
body=f"The PIN of {name} was disabled after {failures} wrong entries ({where}). An Admin can clear it.",
detail={"user_id": str(user_id), "failures": failures, "where": where},
dedupe_minutes=ALERT_DEDUPE_MINUTES,
)
def stepup_session_ended(conn: Any, user_id: Any, failures: int, kind: str, ip: str | None) -> None:
"""A session ended after too many wrong unlock or signature attempts."""
name = _name(conn, user_id)
what = "unlock" if kind == "unlock" else "signature"
notify(
conn,
SECURITY_ALERT,
subject=f"user:{user_id}",
title=f"Session ended after {failures} wrong {what} attempts: {name}",
body=f"{name} entered a wrong PIN or password {failures} times at {what}. The session was ended. "
f"Address: {ip or 'unknown'}.",
detail={"user_id": str(user_id), "failures": failures, "kind": kind, "ip": ip},
dedupe_minutes=ALERT_DEDUPE_MINUTES,
)
# ---------------------------------------------------------------- policy denials
_denied_lock = threading.Lock()
_denied_seen: dict[tuple[str, str, str, str], float] = {}
def _denied_due(key: tuple[str, str, str, str]) -> bool:
now = time.monotonic()
with _denied_lock:
last = _denied_seen.get(key)
if last is not None and now - last < DENIED_DEDUPE_SECONDS:
return False
if len(_denied_seen) > 5000:
_denied_seen.clear()
_denied_seen[key] = now
return True
def record_denied(settings: Settings, principal: Any, *, method: str, path: str, code: str) -> None:
"""Audit `policy.denied` for a 403 refused to a signed-in principal. The method, path and code only (never a
body or a query string). The same user, path and code is audited once a minute at most."""
if code in DENIED_SKIP_CODES:
return
key = (str(principal.user_id), method, path, code)
if not _denied_due(key):
return
from .. import db
try:
with db.get_pool(settings).connection(timeout=5.0) as conn:
conn.autocommit = True
audit.append(
conn,
action="policy.denied",
user_id=principal.user_id,
session_id=principal.session_id,
role_snapshot=principal.role_snapshot,
subject_type="route",
subject_id=f"{method} {path}"[:300],
detail={"method": method, "path": path[:300], "code": code},
ip=principal.ip,
)
except Exception as e:
log.warning("could not audit a denial: %s", type(e).__name__)
# ---------------------------------------------------------------- the Security tab's list
def _row(kind: str, r: dict[str, Any]) -> dict[str, Any]:
return {
"at": r["at"],
"kind": kind,
"user_id": str(r["user_id"]) if r.get("user_id") else None,
"display_name": r.get("display_name"),
"account": r.get("account"),
"ip": r.get("ip"),
"detail": r.get("detail") or {},
}
AUDIT_ALERTS = (
# (kind, WHERE on audit_events e)
("lockout", "e.action = 'auth.lockout'"),
("signin_burst", "e.action = 'auth.signin_burst'"),
("denied", "e.action = 'policy.denied'"),
("admin_change", "e.action LIKE 'admin.%%'"),
("it_edit", "e.it_reason IS NOT NULL"),
)
AUDIT_ALERT_SQL = """
SELECT e.at, e.user_id, u.display_name, coalesce(e.detail->>'account', u.sam_account) AS account,
host(e.ip) AS ip, e.action, e.detail, e.it_reason, e.subject_type, e.subject_id
FROM audit_events e LEFT JOIN users u ON u.id = e.user_id WHERE {where} AND e.at > %s
ORDER BY e.id DESC LIMIT %s
"""
def alerts(conn: Any, hours: int) -> list[dict[str, Any]]:
"""What happened in the last `hours`, newest first, at most ALERTS_MAX rows."""
out: list[dict[str, Any]] = []
since = conn.execute("SELECT now() - make_interval(hours => %s::int)", (hours,)).fetchone()[0]
with conn.cursor(row_factory=dict_row) as cur:
for kind, where in AUDIT_ALERTS:
rows = cur.execute(
AUDIT_ALERT_SQL.format(where=where),
(since, ALERTS_MAX),
).fetchall()
for r in rows:
detail = dict(r["detail"] or {}) if isinstance(r["detail"], dict) else {"value": r["detail"]}
detail.setdefault("action", r["action"])
if r["subject_type"]:
detail.setdefault("subject", f"{r['subject_type']}:{r['subject_id']}")
if kind == "it_edit":
detail["it_reason"] = r["it_reason"]
out.append(_row(kind, {**r, "detail": detail}))
rows = cur.execute(
"SELECT max(a.at) AS at, a.user_id, u.display_name, u.sam_account AS account, count(*) AS n,"
" (array_agg(host(a.ip) ORDER BY a.at DESC))[1] AS ip, array_agg(DISTINCT a.kind) AS kinds"
" FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
" WHERE NOT a.ok AND a.reason = 'bad_pin' AND a.user_id IS NOT NULL AND a.at > %s"
" GROUP BY a.user_id, u.display_name, u.sam_account HAVING count(*) >= %s",
(since, PIN_FAILURES_MIN),
).fetchall()
for r in rows:
out.append(_row("pin_failures", {**r, "detail": {"wrong_pins": int(r["n"]), "where": sorted(r["kinds"])}}))
rows = cur.execute(
"SELECT a.at, a.user_id, u.display_name, u.sam_account AS account, host(a.ip) AS ip, a.kind, a.reason"
" FROM signin_attempts a LEFT JOIN users u ON u.id = a.user_id"
" WHERE NOT a.ok AND a.kind IN ('unlock', 'signature') AND a.reason IN ('bad_pin', 'bad_password')"
" AND a.at > %s ORDER BY a.id DESC LIMIT %s",
(since, ALERTS_MAX),
).fetchall()
for r in rows:
out.append(_row("stepup_wrong_pin", {**r, "detail": {"where": r["kind"], "reason": r["reason"]}}))
rows = cur.execute(
"SELECT f.at, f.kind, f.subject, f.detail, f.run_id FROM integrity_findings f WHERE f.at > %s"
" ORDER BY f.id DESC LIMIT %s",
(since, ALERTS_MAX),
).fetchall()
for r in rows:
detail = {"finding": r["kind"], "subject": r["subject"], "run_id": r["run_id"], **(r["detail"] or {})}
out.append(_row("integrity", {"at": r["at"], "detail": detail}))
out.sort(key=lambda x: x["at"], reverse=True)
return out[:ALERTS_MAX]
+3 -2
View File
@@ -405,14 +405,15 @@ PEOPLE_SQL = """
SELECT u.id, u.display_name, u.welder_stamp, max(CASE r.role::text WHEN 'shop_supervisor' THEN 3
WHEN 'shop_lead' THEN 2 ELSE 1 END) AS level
FROM users u JOIN user_roles r ON r.user_id = u.id
WHERE u.active AND r.dept_code = %s AND r.role::text IN ('shop_worker', 'shop_lead', 'shop_supervisor')
WHERE u.active AND u.profile_active AND r.dept_code = %s
AND r.role::text IN ('shop_worker', 'shop_lead', 'shop_supervisor')
GROUP BY u.id, u.display_name, u.welder_stamp
ORDER BY lower(u.display_name), u.id
"""
def people(conn: Any, dept: str) -> list[dict[str, Any]]:
"""The active users with a shop role in a department, for the welder picker."""
"""The active users with a shop role in a department, for the welder picker. An inactive profile is left out."""
found = packets.rows(conn, PEOPLE_SQL, (dept,))
limits: dict[str, list[int]] = {}
if found: