Phase 9: Admin completion, reconcile, notifications, integrity, backups
Admin: Departments (name and stations; add shows the three groups IT must create), Profiles (welder stamp, station limits, active) with a CSV import that previews and applies only when clean, Checklists (weld and plain), Security (session, lockout and rate-limit settings with bounds, alerts for the last 24 hours, the integrity result), Access log with filters and a guarded CSV, Who is in, Stamp check, Notifications, and the employeeID flags. Every Admin change is audited. Worker: ad-reconcile (nightly and on "refresh from AD now"), session-refresh (15 minutes), integrity (seals, asset hashes, missing and orphan files, the audit chain), backup-check, notify-send and notify-digest. Notifications route per event type to AD groups, users, mailboxes or the department Supervisor group, immediately or in a daily digest, over STARTTLS verified against the PESCO CA, with dry run on the test stack. Backups: scripts/backup/ps-backup.sh and ps-restore.sh with a systemd timer at 02:00 America/Denver. Restore rehearsed on the test stack 2026-10-21. CI: pip-audit and npm audit. pypdf 6.21.1 and certifi 2026.10.19. Migration 0010. Closes the app part of I6, M3, O1, O2 and S3. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,275 @@
|
||||
"""The integrity check, nightly at 04:00 site time.
|
||||
|
||||
- Seals: every unbroken seal of a filed packet is verified again with the seal key.
|
||||
- Asset hashes: the sha256 of every file uploaded in the last 7 days, every night, and of every file on Sundays
|
||||
(site time). Each check sets assets.hash_checked_at and hash_ok.
|
||||
- Missing files: every asset row whose file is not there.
|
||||
- Orphan files: files under ASSETS_DIR that no asset row names. Reported only, never deleted. A file younger than
|
||||
an hour is skipped (an upload in flight).
|
||||
- The audit hash chain, from the first row.
|
||||
|
||||
One integrity_findings row per finding (at most 500 per kind; the counts hold the whole number). Any finding
|
||||
raises the `integrity_failure` condition; a clean, complete run resolves it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import time as clock
|
||||
from datetime import datetime, time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from ... import audit
|
||||
from ...feed.config import FeedSettings
|
||||
from ...settings import SecretFileError
|
||||
from .. import jobs
|
||||
from . import Counts, JobContext, JobSpec, seconds_until
|
||||
|
||||
AT = time(4, 0)
|
||||
NAME = "integrity"
|
||||
KINDS = ("seal", "asset_hash", "asset_missing", "blob_orphan", "audit_chain")
|
||||
CAP = 500
|
||||
RECENT_DAYS = 7
|
||||
ORPHAN_MIN_AGE_SECONDS = 3600
|
||||
CHUNK = 1024 * 1024
|
||||
SKIP_DIRS = {"lost+found"}
|
||||
SUBJECT = "integrity"
|
||||
LABELS = {
|
||||
"seal": "seal does not verify",
|
||||
"asset_hash": "file changed",
|
||||
"asset_missing": "file missing",
|
||||
"blob_orphan": "file with no asset row",
|
||||
"audit_chain": "audit chain break",
|
||||
}
|
||||
|
||||
|
||||
class Findings:
|
||||
"""Collects findings and writes one row each, capped per kind."""
|
||||
|
||||
def __init__(self, conn: Any, run_id: int):
|
||||
self.conn = conn
|
||||
self.run_id = run_id
|
||||
self.counts: dict[str, int] = dict.fromkeys(KINDS, 0)
|
||||
self.samples: dict[str, list[str]] = {k: [] for k in KINDS}
|
||||
|
||||
def add(self, kind: str, subject: str, detail: dict[str, Any]) -> None:
|
||||
self.counts[kind] += 1
|
||||
if self.counts[kind] > CAP:
|
||||
return
|
||||
if len(self.samples[kind]) < 10:
|
||||
self.samples[kind].append(subject)
|
||||
self.conn.execute(
|
||||
"INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, %s, %s, %s)",
|
||||
(self.run_id, kind, subject, Jsonb(detail)),
|
||||
)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return sum(self.counts.values())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- seals
|
||||
|
||||
|
||||
def check_seals(ctx: JobContext, found: Findings) -> int:
|
||||
"""Verify every unbroken seal of a filed packet. Returns the number checked."""
|
||||
from ...domain import packets
|
||||
from ...domain.pipeline import seal_hash, seal_key_id
|
||||
|
||||
conn = ctx.conn
|
||||
rows = conn.execute(
|
||||
"SELECT s.id, s.packet_id, s.hash, s.key_id FROM packet_seals s JOIN packets p ON p.id = s.packet_id"
|
||||
" WHERE s.broken_at IS NULL AND p.status = 'filed' ORDER BY s.packet_id"
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
try:
|
||||
key = ctx.settings.seal_key()
|
||||
except SecretFileError as e:
|
||||
ctx.degrade(f"seals not checked: {e}")
|
||||
return 0
|
||||
kid = seal_key_id(key)
|
||||
checked = 0
|
||||
for seal_id, packet_id, digest, key_id in rows:
|
||||
detail: dict[str, Any] = {"packet_id": packet_id, "seal_id": seal_id}
|
||||
if key_id != kid:
|
||||
found.add("seal", f"packet:{packet_id}", {**detail, "reason": "sealed with another key", "key_id": key_id})
|
||||
checked += 1
|
||||
continue
|
||||
try:
|
||||
with conn.transaction():
|
||||
row = packets.load_row(conn, packet_id)
|
||||
content = packets.seal_content(conn, row) if row is not None else None
|
||||
except Exception as e: # one packet that cannot be read does not stop the others
|
||||
ctx.error(f"packet {packet_id}: seal not checked: {type(e).__name__}: {e}")
|
||||
continue
|
||||
if content is None:
|
||||
continue
|
||||
ok = hmac.compare_digest(seal_hash(key, content), bytes(digest))
|
||||
if not ok:
|
||||
found.add("seal", f"packet:{packet_id}", {**detail, "reason": "the packet no longer matches its seal"})
|
||||
checked += 1
|
||||
return checked
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- assets
|
||||
|
||||
|
||||
def _path(root: Path, key: str) -> Path | None:
|
||||
p = (root / key).resolve()
|
||||
return p if p.is_relative_to(root) else None
|
||||
|
||||
|
||||
def sha256_of(path: Path) -> bytes:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
h.update(chunk)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def check_assets(ctx: JobContext, found: Findings, *, full: bool) -> dict[str, int]:
|
||||
"""Missing files for every asset row; the sha256 of the recent ones (all of them when full)."""
|
||||
conn = ctx.conn
|
||||
root = Path(ctx.settings.assets_dir).resolve()
|
||||
rows = conn.execute(
|
||||
"SELECT id, storage_key, sha256, size, uploaded_at >= now() - make_interval(days => %s) AS recent"
|
||||
" FROM assets ORDER BY uploaded_at, id",
|
||||
(RECENT_DAYS,),
|
||||
).fetchall()
|
||||
hashed = 0
|
||||
for aid, key, digest, size, recent in rows:
|
||||
p = _path(root, str(key))
|
||||
if p is None or not p.is_file():
|
||||
found.add("asset_missing", str(aid), {"storage_key": key})
|
||||
conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
|
||||
continue
|
||||
if not (full or recent):
|
||||
continue
|
||||
try:
|
||||
actual = sha256_of(p)
|
||||
except OSError as e:
|
||||
found.add("asset_missing", str(aid), {"storage_key": key, "error": e.strerror or type(e).__name__})
|
||||
conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
|
||||
continue
|
||||
ok = hmac.compare_digest(actual, bytes(digest))
|
||||
hashed += 1
|
||||
if not ok:
|
||||
found.add(
|
||||
"asset_hash",
|
||||
str(aid),
|
||||
{"storage_key": key, "expected": bytes(digest).hex(), "actual": actual.hex(), "size": size},
|
||||
)
|
||||
conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = %s WHERE id = %s", (ok, aid))
|
||||
return {"assets": len(rows), "hashed": hashed}
|
||||
|
||||
|
||||
def check_orphans(ctx: JobContext, found: Findings) -> int:
|
||||
"""Files under ASSETS_DIR that no asset row names. Never deletes anything. Returns the files looked at."""
|
||||
root = Path(ctx.settings.assets_dir)
|
||||
if not root.is_dir():
|
||||
ctx.degrade(f"ASSETS_DIR {root} is not a folder")
|
||||
return 0
|
||||
keys = {str(r[0]) for r in ctx.conn.execute("SELECT storage_key FROM assets")}
|
||||
cutoff = clock.time() - ORPHAN_MIN_AGE_SECONDS
|
||||
seen = 0
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
||||
for name in filenames:
|
||||
p = Path(dirpath) / name
|
||||
rel = p.relative_to(root).as_posix()
|
||||
seen += 1
|
||||
if rel in keys:
|
||||
continue
|
||||
try:
|
||||
st = p.stat()
|
||||
except OSError:
|
||||
continue
|
||||
if st.st_mtime > cutoff:
|
||||
continue
|
||||
found.add(
|
||||
"blob_orphan", rel, {"size": st.st_size, "mtime": datetime.fromtimestamp(st.st_mtime).isoformat()}
|
||||
)
|
||||
return seen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- audit chain
|
||||
|
||||
_ROW = re.compile(r"^row (\d+):")
|
||||
|
||||
|
||||
def check_audit(ctx: JobContext, found: Findings) -> int:
|
||||
ok, n, detail = audit.verify_chain(ctx.conn)
|
||||
if not ok:
|
||||
m = _ROW.match(detail)
|
||||
found.add("audit_chain", m.group(1) if m else "?", {"detail": detail, "rows_checked": n})
|
||||
return n
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the run
|
||||
|
||||
|
||||
def _event_text(found: Findings) -> tuple[str, str]:
|
||||
parts = [f"{n} {LABELS[k]}" for k, n in found.counts.items() if n]
|
||||
lines = []
|
||||
for k, n in found.counts.items():
|
||||
if n:
|
||||
lines.append(f"{LABELS[k].capitalize()}: {n} ({', '.join(found.samples[k])}{', ...' if n > 10 else ''}).")
|
||||
lines.append("Admin, Security tab: the full list of findings.")
|
||||
title = f"Integrity check found {found.total} problem{'s' if found.total != 1 else ''}: " + ", ".join(parts)
|
||||
return title, "\n".join(lines)
|
||||
|
||||
|
||||
def full_day(now_site: datetime) -> bool:
|
||||
"""Sunday, site time: every asset is rehashed."""
|
||||
return now_site.weekday() == 6
|
||||
|
||||
|
||||
def run(ctx: JobContext) -> Counts:
|
||||
conn = ctx.conn
|
||||
full = full_day(ctx.now_site())
|
||||
found = Findings(conn, ctx.run_id)
|
||||
counts: Counts = {"full": full}
|
||||
with conn.transaction():
|
||||
counts["seals"] = check_seals(ctx, found)
|
||||
with conn.transaction():
|
||||
counts.update(check_assets(ctx, found, full=full))
|
||||
with conn.transaction():
|
||||
counts["files"] = check_orphans(ctx, found)
|
||||
with conn.transaction():
|
||||
counts["audit_rows"] = check_audit(ctx, found)
|
||||
counts["findings"] = dict(found.counts)
|
||||
counts["total"] = found.total
|
||||
try:
|
||||
with conn.transaction():
|
||||
if found.total:
|
||||
title, body = _event_text(found)
|
||||
jobs.raise_event(
|
||||
conn,
|
||||
"integrity_failure",
|
||||
subject=SUBJECT,
|
||||
title=title,
|
||||
body=body,
|
||||
detail={"run_id": ctx.run_id, **found.counts},
|
||||
severity="critical",
|
||||
condition=True,
|
||||
)
|
||||
elif not ctx.degraded:
|
||||
jobs.resolve_event(conn, "integrity_failure", subject=SUBJECT, title="Integrity check failure")
|
||||
except Exception as e:
|
||||
ctx.log.exception("integrity notification failed")
|
||||
ctx.error(f"notification: {type(e).__name__}: {e}", degrade=False)
|
||||
return counts
|
||||
|
||||
|
||||
def cadence(_: FeedSettings, now_site: datetime) -> int:
|
||||
return seconds_until(now_site, AT)
|
||||
|
||||
|
||||
SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)
|
||||
Reference in New Issue
Block a user