"""The integrity check, nightly at 04:00 site time. - Seals: every unbroken seal of a filed packet is verified again with the seal key. - Asset hashes: the sha256 of every file uploaded in the last 7 days, every night, and of every file on Sundays (site time). Each check sets assets.hash_checked_at and hash_ok. - Missing files: every asset row whose file is not there. - Orphan files: files under ASSETS_DIR that no asset row names. Reported only, never deleted. A file younger than an hour is skipped (an upload in flight). - The audit hash chain, from the first row. One integrity_findings row per finding (at most 500 per kind; the counts hold the whole number). Any finding raises the `integrity_failure` condition; a clean, complete run resolves it. """ from __future__ import annotations import hashlib import hmac import os import re import time as clock from datetime import datetime, time from pathlib import Path from typing import Any from psycopg.types.json import Jsonb from ... import audit from ...feed.config import FeedSettings from ...settings import SecretFileError from .. import jobs from . import Counts, JobContext, JobSpec, seconds_until AT = time(4, 0) NAME = "integrity" KINDS = ("seal", "asset_hash", "asset_missing", "blob_orphan", "audit_chain") CAP = 500 RECENT_DAYS = 7 ORPHAN_MIN_AGE_SECONDS = 3600 CHUNK = 1024 * 1024 SKIP_DIRS = {"lost+found"} SUBJECT = "integrity" LABELS = { "seal": "seal does not verify", "asset_hash": "file changed", "asset_missing": "file missing", "blob_orphan": "file with no asset row", "audit_chain": "audit chain break", } class Findings: """Collects findings and writes one row each, capped per kind.""" def __init__(self, conn: Any, run_id: int): self.conn = conn self.run_id = run_id self.counts: dict[str, int] = dict.fromkeys(KINDS, 0) self.samples: dict[str, list[str]] = {k: [] for k in KINDS} def add(self, kind: str, subject: str, detail: dict[str, Any]) -> None: self.counts[kind] += 1 if self.counts[kind] > CAP: return if len(self.samples[kind]) < 10: self.samples[kind].append(subject) self.conn.execute( "INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, %s, %s, %s)", (self.run_id, kind, subject, Jsonb(detail)), ) @property def total(self) -> int: return sum(self.counts.values()) # ---------------------------------------------------------------- seals def check_seals(ctx: JobContext, found: Findings) -> int: """Verify every unbroken seal of a filed packet. Returns the number checked.""" from ...domain import packets from ...domain.pipeline import seal_hash, seal_key_id conn = ctx.conn rows = conn.execute( "SELECT s.id, s.packet_id, s.hash, s.key_id FROM packet_seals s JOIN packets p ON p.id = s.packet_id" " WHERE s.broken_at IS NULL AND p.status = 'filed' ORDER BY s.packet_id" ).fetchall() if not rows: return 0 try: key = ctx.settings.seal_key() except SecretFileError as e: ctx.degrade(f"seals not checked: {e}") return 0 kid = seal_key_id(key) checked = 0 for seal_id, packet_id, digest, key_id in rows: detail: dict[str, Any] = {"packet_id": packet_id, "seal_id": seal_id} if key_id != kid: found.add("seal", f"packet:{packet_id}", {**detail, "reason": "sealed with another key", "key_id": key_id}) checked += 1 continue try: with conn.transaction(): row = packets.load_row(conn, packet_id) content = packets.seal_content(conn, row) if row is not None else None except Exception as e: # one packet that cannot be read does not stop the others ctx.error(f"packet {packet_id}: seal not checked: {type(e).__name__}: {e}") continue if content is None: continue ok = hmac.compare_digest(seal_hash(key, content), bytes(digest)) if not ok: found.add("seal", f"packet:{packet_id}", {**detail, "reason": "the packet no longer matches its seal"}) checked += 1 return checked # ---------------------------------------------------------------- assets def _path(root: Path, key: str) -> Path | None: p = (root / key).resolve() return p if p.is_relative_to(root) else None def sha256_of(path: Path) -> bytes: h = hashlib.sha256() with path.open("rb") as f: while chunk := f.read(CHUNK): h.update(chunk) return h.digest() def check_assets(ctx: JobContext, found: Findings, *, full: bool) -> dict[str, int]: """Missing files for every asset row; the sha256 of the recent ones (all of them when full).""" conn = ctx.conn root = Path(ctx.settings.assets_dir).resolve() rows = conn.execute( "SELECT id, storage_key, sha256, size, uploaded_at >= now() - make_interval(days => %s) AS recent" " FROM assets ORDER BY uploaded_at, id", (RECENT_DAYS,), ).fetchall() hashed = 0 for aid, key, digest, size, recent in rows: p = _path(root, str(key)) if p is None or not p.is_file(): found.add("asset_missing", str(aid), {"storage_key": key}) conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,)) continue if not (full or recent): continue try: actual = sha256_of(p) except OSError as e: found.add("asset_missing", str(aid), {"storage_key": key, "error": e.strerror or type(e).__name__}) conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,)) continue ok = hmac.compare_digest(actual, bytes(digest)) hashed += 1 if not ok: found.add( "asset_hash", str(aid), {"storage_key": key, "expected": bytes(digest).hex(), "actual": actual.hex(), "size": size}, ) conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = %s WHERE id = %s", (ok, aid)) return {"assets": len(rows), "hashed": hashed} def check_orphans(ctx: JobContext, found: Findings) -> int: """Files under ASSETS_DIR that no asset row names. Never deletes anything. Returns the files looked at.""" root = Path(ctx.settings.assets_dir) if not root.is_dir(): ctx.degrade(f"ASSETS_DIR {root} is not a folder") return 0 keys = {str(r[0]) for r in ctx.conn.execute("SELECT storage_key FROM assets")} cutoff = clock.time() - ORPHAN_MIN_AGE_SECONDS seen = 0 for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] for name in filenames: p = Path(dirpath) / name rel = p.relative_to(root).as_posix() seen += 1 if rel in keys: continue try: st = p.stat() except OSError: continue if st.st_mtime > cutoff: continue found.add( "blob_orphan", rel, {"size": st.st_size, "mtime": datetime.fromtimestamp(st.st_mtime).isoformat()} ) return seen # ---------------------------------------------------------------- audit chain _ROW = re.compile(r"^row (\d+):") def check_audit(ctx: JobContext, found: Findings) -> int: ok, n, detail = audit.verify_chain(ctx.conn) if not ok: m = _ROW.match(detail) found.add("audit_chain", m.group(1) if m else "?", {"detail": detail, "rows_checked": n}) return n # ---------------------------------------------------------------- the run def _event_text(found: Findings) -> tuple[str, str]: parts = [f"{n} {LABELS[k]}" for k, n in found.counts.items() if n] lines = [] for k, n in found.counts.items(): if n: lines.append(f"{LABELS[k].capitalize()}: {n} ({', '.join(found.samples[k])}{', ...' if n > 10 else ''}).") lines.append("Admin, Security tab: the full list of findings.") title = f"Integrity check found {found.total} problem{'s' if found.total != 1 else ''}: " + ", ".join(parts) return title, "\n".join(lines) def full_day(now_site: datetime) -> bool: """Sunday, site time: every asset is rehashed.""" return now_site.weekday() == 6 def run(ctx: JobContext) -> Counts: conn = ctx.conn full = full_day(ctx.now_site()) found = Findings(conn, ctx.run_id) counts: Counts = {"full": full} with conn.transaction(): counts["seals"] = check_seals(ctx, found) with conn.transaction(): counts.update(check_assets(ctx, found, full=full)) with conn.transaction(): counts["files"] = check_orphans(ctx, found) with conn.transaction(): counts["audit_rows"] = check_audit(ctx, found) counts["findings"] = dict(found.counts) counts["total"] = found.total try: with conn.transaction(): if found.total: title, body = _event_text(found) jobs.raise_event( conn, "integrity_failure", subject=SUBJECT, title=title, body=body, detail={"run_id": ctx.run_id, **found.counts}, severity="critical", condition=True, ) elif not ctx.degraded: jobs.resolve_event(conn, "integrity_failure", subject=SUBJECT, title="Integrity check failure") except Exception as e: ctx.log.exception("integrity notification failed") ctx.error(f"notification: {type(e).__name__}: {e}", degrade=False) return counts def cadence(_: FeedSettings, now_site: datetime) -> int: return seconds_until(now_site, AT) SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)