Phase 2: core domain, policy engine, packet API, Clerical screen

Migration 0003: projects, aliases, renumbers, units, packets and every child table, change orders, assets, documents, ITPs, station limits, department weld flag. Policy package: Principal, PacketState, Action rules, custody, preconditions, CO hold, order of checks, can/blocked/holder on reads. docs/POLICY.md with the tightenings table for Seth. Write pipeline: FOR UPDATE, CAS on status and dept, If-Match row versions, server-assigned numbers, field CAS, version bump, packet log, audit, NOTIFY, signatures, IT reasons, filed lock and HMAC seal. API: /api/packets (header, transitions, work orders, pages, docs, move, remove, BOM, shorts, welds, heat tags, heat table, stage, checklist, clock, warehouse stage, Q.C., change orders, log, seal), /api/projects, /api/units, /api/assets, /_blob. Frontend: Clerical home, projects and units, packet page with pdf.js and QR codes, hand draft, heat map, send to warehouse, department or Q.C. Tests: policy matrix, API, concurrency, schema, projects; Playwright Clerical draft scenario. Verified on docker-1 with the CI stack and the Samba DC. Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 09:31:45 -06:00
parent 97b0973130
commit 37d9a0cb50
83 changed files with 16490 additions and 55 deletions
@@ -0,0 +1,231 @@
+"""Stored files. One row per file in `assets`; the bytes live at ASSETS_DIR/<id[:2]>/<id>.
+
+An upload is streamed to a temp file in ASSETS_DIR in chunks (never held in memory whole), hashed as it goes,
+typed by its first bytes (not by its name or the browser's claim), fsynced and renamed into place. The same bytes
+uploaded twice (same sha256 and size) return the existing row.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import logging
+import os
+import re
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from psycopg.rows import dict_row
+
+from .. import audit
+from ..api.errors import ApiError
+from ..settings import Settings
+from ..text import clean_text
+
+log = logging.getLogger(__name__)
+
+CHUNK = 1024 * 1024
+HEAD = 16
+PDF = "application/pdf"
+MAGIC: tuple[tuple[bytes, str], ...] = (
+    (b"%PDF-", PDF),
+    (b"\x89PNG\r\n\x1a\n", "image/png"),
+    (b"\xff\xd8\xff", "image/jpeg"),
+)
+EXT = {PDF: ".pdf", "image/png": ".png", "image/jpeg": ".jpg"}
+NAME_MAX = 160
+_UNSAFE_NAME = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
+
+
+def sniff(head: bytes) -> str | None:
+    """The mime type from the first bytes, or None when it is not a type Packet Station stores."""
+    for magic, mime in MAGIC:
+        if head.startswith(magic):
+            return mime
+    return None
+
+
+def safe_name(name: Any) -> str | None:
+    """The original file name, cleaned: no path, no separators or reserved characters, at most 160 characters."""
+    text = clean_text(name, 1000)
+    text = text.replace("\\", "/").rsplit("/", 1)[-1]
+    text = _UNSAFE_NAME.sub("_", text).strip(" .")
+    return text[:NAME_MAX] or None
+
+
+def storage_key(asset_id: uuid.UUID | str) -> str:
+    s = str(asset_id)
+    return f"{s[:2]}/{s}"
+
+
+def path_of(settings: Settings, key: str) -> Path:
+    root = Path(settings.assets_dir).resolve()
+    p = (root / key).resolve()
+    if not p.is_relative_to(root):
+        raise ApiError(404, "not_found", "No such file.")
+    return p
+
+
+def pdf_pages(path: Path) -> int:
+    """Page count through pypdf; 422 bad_pdf when it does not parse or has no pages."""
+    from pypdf import PdfReader
+    from pypdf.errors import PyPdfError
+
+    try:
+        n = len(PdfReader(str(path)).pages)
+    except (PyPdfError, ValueError, KeyError, TypeError, OSError, IndexError, AttributeError) as e:
+        log.info("upload is not a readable PDF: %s", type(e).__name__)
+        raise ApiError(422, "bad_pdf", "That PDF could not be read.") from None
+    if n < 1:
+        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
+    return n
+
+
+def public(row: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": str(row["id"]),
+        "mime": row["mime"],
+        "size": int(row["size"]),
+        "sha256": bytes(row["sha256"]).hex(),
+        "pages": row["pages"],
+        "original_name": row["original_name"],
+        "url": f"/_blob/{row['id']}",
+    }
+
+
+def get(conn: Any, asset_id: Any) -> dict[str, Any] | None:
+    try:
+        aid = uuid.UUID(str(asset_id))
+    except ValueError:
+        return None
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute("SELECT * FROM assets WHERE id = %s", (aid,)).fetchone()
+    return dict(r) if r else None
+
+
+def _fsync_dir(path: Path) -> None:
+    if os.name != "posix":
+        return
+    fd = os.open(str(path), os.O_RDONLY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def store_upload(
+    conn: Any,
+    settings: Settings,
+    stream: BinaryIO,
+    *,
+    original_name: Any,
+    user: Any,
+    packet_id: int | None = None,
+    source: str = "upload",
+    source_ref: Any = None,
+    actor: Any = None,
+) -> dict[str, Any]:
+    """Store one upload and return its public row. Raises 413 too_large, 415 unsupported_type, 422 bad_pdf.
+
+    `user` is the uploader's user id; `actor` (the session Principal) is used for the audit row when given.
+    """
+    root = Path(settings.assets_dir)
+    root.mkdir(parents=True, exist_ok=True)
+    limit = int(settings.upload_max_bytes)
+    fd, tmp_name = tempfile.mkstemp(prefix=".up-", dir=str(root))
+    tmp = Path(tmp_name)
+    keep = False
+    placed: Path | None = None
+    try:
+        digest = hashlib.sha256()
+        size = 0
+        head = b""
+        mime: str | None = None
+        with os.fdopen(fd, "wb") as out:
+            while True:
+                chunk = stream.read(CHUNK)
+                if not chunk:
+                    break
+                size += len(chunk)
+                if size > limit:
+                    raise ApiError(413, "too_large", f"The file is over the {limit // (1024 * 1024)} MB limit.")
+                if mime is None and len(head) < HEAD:
+                    head += chunk[: HEAD - len(head)]
+                    if len(head) >= HEAD:
+                        mime = sniff(head)
+                        if mime is None:
+                            raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.")
+                digest.update(chunk)
+                out.write(chunk)
+            out.flush()
+            os.fsync(out.fileno())
+        if mime is None:
+            mime = sniff(head)
+        if mime is None or size == 0:
+            raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.")
+        sha = digest.digest()
+        with conn.cursor(row_factory=dict_row) as cur:
+            existing = cur.execute(
+                "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
+            ).fetchone()
+        if existing is not None:
+            return public(dict(existing))
+        pages = pdf_pages(tmp) if mime == PDF else None
+        aid = uuid.uuid4()
+        key = storage_key(aid)
+        final = path_of(settings, key)
+        final.parent.mkdir(parents=True, exist_ok=True)
+        os.replace(tmp, final)
+        keep = True
+        placed = final
+        _fsync_dir(final.parent)
+        name = safe_name(original_name)
+        with conn.transaction():
+            with conn.cursor(row_factory=dict_row) as cur:
+                row = cur.execute(
+                    "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
+                    " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
+                    (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id),
+                ).fetchone()
+            audit.append(
+                conn,
+                action="asset.uploaded",
+                user_id=user,
+                session_id=getattr(actor, "session_id", None),
+                role_snapshot=getattr(actor, "role_snapshot", None),
+                subject_type="asset",
+                subject_id=str(aid),
+                packet_id=packet_id,
+                detail={"mime": mime, "size": size, "pages": pages, "sha256": sha.hex()},
+                ip=getattr(actor, "ip", None),
+            )
+        return public(dict(row))
+    except BaseException:
+        if placed is not None:  # the row was not written: do not leave an orphan file
+            with contextlib.suppress(OSError):
+                placed.unlink()
+        raise
+    finally:
+        if not keep:
+            with contextlib.suppress(OSError):
+                tmp.unlink()
+
+
+def open_blob(settings: Settings, row: dict[str, Any]) -> Path:
+    """The stored file of an asset row; 404 when it is missing on disk."""
+    p = path_of(settings, str(row["storage_key"]))
+    if not p.is_file():
+        log.error("asset %s is missing on disk", row["id"])
+        raise ApiError(404, "not_found", "No such file.")
+    return p
+
+
+def download_name(row: dict[str, Any]) -> str:
+    """A safe file name for Content-Disposition (ASCII only, quotes removed)."""
+    name = row.get("original_name") or ""
+    name = name.encode("ascii", "ignore").decode("ascii").replace('"', "").replace(";", "_").strip()
+    if not name:
+        name = str(row["id"]) + EXT.get(str(row["mime"]), "")
+    return name