PacketStation/server/packetstation/worker/jobs/integrity.py

"""The integrity check, nightly at 04:00 site time.

- Seals: every unbroken seal of a filed packet is verified again with the seal key.
- Asset hashes: the sha256 of every file uploaded in the last 7 days, every night, and of every file on Sundays
  (site time). Each check sets assets.hash_checked_at and hash_ok.
- Missing files: every asset row whose file is not there.
- Orphan files: files under ASSETS_DIR that no asset row names. Reported only, never deleted. A file younger than
  an hour is skipped (an upload in flight).
- The audit hash chain, from the first row.

One integrity_findings row per finding (at most 500 per kind; the counts hold the whole number). Any finding
raises the `integrity_failure` condition; a clean, complete run resolves it.
"""

from __future__ import annotations

import hashlib
import hmac
import os
import re
import time as clock
from datetime import datetime, time
from pathlib import Path
from typing import Any

from psycopg.types.json import Jsonb

from ... import audit
from ...feed.config import FeedSettings
from ...settings import SecretFileError
from .. import jobs
from . import Counts, JobContext, JobSpec, seconds_until

AT = time(4, 0)
NAME = "integrity"
KINDS = ("seal", "asset_hash", "asset_missing", "blob_orphan", "audit_chain")
CAP = 500
RECENT_DAYS = 7
ORPHAN_MIN_AGE_SECONDS = 3600
CHUNK = 1024 * 1024
SKIP_DIRS = {"lost+found"}
SUBJECT = "integrity"
LABELS = {
    "seal": "seal does not verify",
    "asset_hash": "file changed",
    "asset_missing": "file missing",
    "blob_orphan": "file with no asset row",
    "audit_chain": "audit chain break",
}


class Findings:
    """Collects findings and writes one row each, capped per kind."""

    def __init__(self, conn: Any, run_id: int):
        self.conn = conn
        self.run_id = run_id
        self.counts: dict[str, int] = dict.fromkeys(KINDS, 0)
        self.samples: dict[str, list[str]] = {k: [] for k in KINDS}

    def add(self, kind: str, subject: str, detail: dict[str, Any]) -> None:
        self.counts[kind] += 1
        if self.counts[kind] > CAP:
            return
        if len(self.samples[kind]) < 10:
            self.samples[kind].append(subject)
        self.conn.execute(
            "INSERT INTO integrity_findings (run_id, kind, subject, detail) VALUES (%s, %s, %s, %s)",
            (self.run_id, kind, subject, Jsonb(detail)),
        )

    @property
    def total(self) -> int:
        return sum(self.counts.values())


# ---------------------------------------------------------------- seals


def check_seals(ctx: JobContext, found: Findings) -> int:
    """Verify every unbroken seal of a filed packet. Returns the number checked."""
    from ...domain import packets
    from ...domain.pipeline import seal_hash, seal_key_id

    conn = ctx.conn
    rows = conn.execute(
        "SELECT s.id, s.packet_id, s.hash, s.key_id FROM packet_seals s JOIN packets p ON p.id = s.packet_id"
        " WHERE s.broken_at IS NULL AND p.status = 'filed' ORDER BY s.packet_id"
    ).fetchall()
    if not rows:
        return 0
    try:
        key = ctx.settings.seal_key()
    except SecretFileError as e:
        ctx.degrade(f"seals not checked: {e}")
        return 0
    kid = seal_key_id(key)
    checked = 0
    for seal_id, packet_id, digest, key_id in rows:
        detail: dict[str, Any] = {"packet_id": packet_id, "seal_id": seal_id}
        if key_id != kid:
            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "sealed with another key", "key_id": key_id})
            checked += 1
            continue
        try:
            with conn.transaction():
                row = packets.load_row(conn, packet_id)
                content = packets.seal_content(conn, row) if row is not None else None
        except Exception as e:  # one packet that cannot be read does not stop the others
            ctx.error(f"packet {packet_id}: seal not checked: {type(e).__name__}: {e}")
            continue
        if content is None:
            continue
        ok = hmac.compare_digest(seal_hash(key, content), bytes(digest))
        if not ok:
            found.add("seal", f"packet:{packet_id}", {**detail, "reason": "the packet no longer matches its seal"})
        checked += 1
    return checked


# ---------------------------------------------------------------- assets


def _path(root: Path, key: str) -> Path | None:
    p = (root / key).resolve()
    return p if p.is_relative_to(root) else None


def sha256_of(path: Path) -> bytes:
    h = hashlib.sha256()
    with path.open("rb") as f:
        while chunk := f.read(CHUNK):
            h.update(chunk)
    return h.digest()


def check_assets(ctx: JobContext, found: Findings, *, full: bool) -> dict[str, int]:
    """Missing files for every asset row; the sha256 of the recent ones (all of them when full)."""
    conn = ctx.conn
    root = Path(ctx.settings.assets_dir).resolve()
    rows = conn.execute(
        "SELECT id, storage_key, sha256, size, uploaded_at >= now() - make_interval(days => %s) AS recent"
        " FROM assets ORDER BY uploaded_at, id",
        (RECENT_DAYS,),
    ).fetchall()
    hashed = 0
    for aid, key, digest, size, recent in rows:
        p = _path(root, str(key))
        if p is None or not p.is_file():
            found.add("asset_missing", str(aid), {"storage_key": key})
            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
            continue
        if not (full or recent):
            continue
        try:
            actual = sha256_of(p)
        except OSError as e:
            found.add("asset_missing", str(aid), {"storage_key": key, "error": e.strerror or type(e).__name__})
            conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = false WHERE id = %s", (aid,))
            continue
        ok = hmac.compare_digest(actual, bytes(digest))
        hashed += 1
        if not ok:
            found.add(
                "asset_hash",
                str(aid),
                {"storage_key": key, "expected": bytes(digest).hex(), "actual": actual.hex(), "size": size},
            )
        conn.execute("UPDATE assets SET hash_checked_at = now(), hash_ok = %s WHERE id = %s", (ok, aid))
    return {"assets": len(rows), "hashed": hashed}


def check_orphans(ctx: JobContext, found: Findings) -> int:
    """Files under ASSETS_DIR that no asset row names. Never deletes anything. Returns the files looked at."""
    root = Path(ctx.settings.assets_dir)
    if not root.is_dir():
        ctx.degrade(f"ASSETS_DIR {root} is not a folder")
        return 0
    keys = {str(r[0]) for r in ctx.conn.execute("SELECT storage_key FROM assets")}
    cutoff = clock.time() - ORPHAN_MIN_AGE_SECONDS
    seen = 0
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
        for name in filenames:
            p = Path(dirpath) / name
            rel = p.relative_to(root).as_posix()
            seen += 1
            if rel in keys:
                continue
            try:
                st = p.stat()
            except OSError:
                continue
            if st.st_mtime > cutoff:
                continue
            found.add(
                "blob_orphan", rel, {"size": st.st_size, "mtime": datetime.fromtimestamp(st.st_mtime).isoformat()}
            )
    return seen


# ---------------------------------------------------------------- audit chain

_ROW = re.compile(r"^row (\d+):")


def check_audit(ctx: JobContext, found: Findings) -> int:
    ok, n, detail = audit.verify_chain(ctx.conn)
    if not ok:
        m = _ROW.match(detail)
        found.add("audit_chain", m.group(1) if m else "?", {"detail": detail, "rows_checked": n})
    return n


# ---------------------------------------------------------------- the run


def _event_text(found: Findings) -> tuple[str, str]:
    parts = [f"{n} {LABELS[k]}" for k, n in found.counts.items() if n]
    lines = []
    for k, n in found.counts.items():
        if n:
            lines.append(f"{LABELS[k].capitalize()}: {n} ({', '.join(found.samples[k])}{', ...' if n > 10 else ''}).")
    lines.append("Admin, Security tab: the full list of findings.")
    title = f"Integrity check found {found.total} problem{'s' if found.total != 1 else ''}: " + ", ".join(parts)
    return title, "\n".join(lines)


def full_day(now_site: datetime) -> bool:
    """Sunday, site time: every asset is rehashed."""
    return now_site.weekday() == 6


def run(ctx: JobContext) -> Counts:
    conn = ctx.conn
    full = full_day(ctx.now_site())
    found = Findings(conn, ctx.run_id)
    counts: Counts = {"full": full}
    with conn.transaction():
        counts["seals"] = check_seals(ctx, found)
    with conn.transaction():
        counts.update(check_assets(ctx, found, full=full))
    with conn.transaction():
        counts["files"] = check_orphans(ctx, found)
    with conn.transaction():
        counts["audit_rows"] = check_audit(ctx, found)
    counts["findings"] = dict(found.counts)
    counts["total"] = found.total
    try:
        with conn.transaction():
            if found.total:
                title, body = _event_text(found)
                jobs.raise_event(
                    conn,
                    "integrity_failure",
                    subject=SUBJECT,
                    title=title,
                    body=body,
                    detail={"run_id": ctx.run_id, **found.counts},
                    severity="critical",
                    condition=True,
                )
            elif not ctx.degraded:
                jobs.resolve_event(conn, "integrity_failure", subject=SUBJECT, title="Integrity check failure")
    except Exception as e:
        ctx.log.exception("integrity notification failed")
        ctx.error(f"notification: {type(e).__name__}: {e}", degrade=False)
    return counts


def cadence(_: FeedSettings, now_site: datetime) -> int:
    return seconds_until(now_site, AT)


SPEC = JobSpec(name=NAME, run=run, cadence=cadence, pausable=False)