Phase 7: close, file, PDFs and prints

Clerical close and file: the VISUAL observed state beside the close confirm, a 409 with a second confirm when the
feed still shows work orders open, filing with a signature and the HMAC seal, the data book built on the server
and stored on the seal in the filing transaction, admin reopen keeping the broken seal with its data book.

Server PDFs with reportlab, pypdf and pypdfium2: the manufacturing data book page for page after the old build
(cover, index, drawings with VOID stamps, traceability and heat map rasters, weld maps and welding records, Q.C.
inspections, Q.C. and test documents, MTRs, record history), the job traveler with work order QR codes, the ITP
PDF filed automatically at sign-off, every print kind with the PRINTED COPY stamp, the controlled-copy stamp
signed with the seal key, PDF metadata, the traceability CSV with the formula guard, and the append-only
print_log with an audit row for every print and export (migration 0008).

Frontend: the close and file panel, the print menu and its entry points, the printed copies list on the History
tab, the ITP PDF buttons. Tests: server (close, file, reopen, seal, the builders, prints, CSV, stamps), frontend,
and the Playwright close and print scenarios. Docs updated; SECURITY closes M2, S6, S7 and S8.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/domain/packets.py

@@ -47,6 +47,9 @@ SEAL_EXCLUDE = (
     "qc_folder_docs",
     "itp",
     "itp_hold",
+    "seals",
+    "vclose_check",
+    "prints",
 )
 
 
@@ -676,6 +679,92 @@ def visual_state(conn: Any, row: Mapping[str, Any]) -> dict[str, Any] | None:
     }
 
 
+# Leg status codes the feed records that mean the work order is no longer open in VISUAL: closed and cancelled.
+LEG_DONE = ("C", "X")
+
+
+def open_wos(vs: Mapping[str, Any] | None) -> list[str]:
+    """The work orders the feed last saw open in VISUAL, as "wo_base/lot" text, in leg order without repeats.
+
+    A leg is open when the feed recorded a status for it and that status is not closed (C) or cancelled (X). A leg
+    with no recorded status is left out: the feed has not seen it. The IDs stay text exactly as stored."""
+    out: list[str] = []
+    for leg in (vs or {}).get("legs") or []:
+        if not isinstance(leg, Mapping):
+            continue
+        st = str(leg.get("st") or leg.get("status") or "").strip().upper()
+        if not st or st in LEG_DONE:
+            continue
+        parts = str(leg.get("key") or "").split("/")
+        if len(parts) >= 4:
+            base, lot = "/".join(parts[:-3]), parts[-3]
+        else:
+            base, lot = parts[0], (parts[1] if len(parts) > 1 else "")
+        if not base:
+            continue
+        text = f"{base}/{lot}" if lot else base
+        if text not in out:
+            out.append(text)
+    return out
+
+
+def vclose_check(row: Mapping[str, Any], vs: Mapping[str, Any] | None) -> dict[str, Any] | None:
+    """What the feed says about closing the job in VISUAL, next to the Clerical confirm. None for a hand packet.
+
+    `agrees` is true when the Clerical confirm (vclosed) and the feed's closed time are both set or both unset."""
+    if vs is None:
+        return None
+    closed_at = vs.get("closed_at")
+    return {
+        "observed_at": vs.get("observed_at"),
+        "closed_at": closed_at,
+        "open_wos": open_wos(vs),
+        "agrees": (row.get("vclosed_at") is not None) == (closed_at is not None),
+    }
+
+
+def packet_open_wos(conn: Any, row: Mapping[str, Any]) -> list[str]:
+    """The open work orders of `vclose_check` for one packets row (empty for a hand packet)."""
+    return open_wos(visual_state(conn, row))
+
+
+def seal_count(conn: Any, packet_id: int) -> int:
+    r = conn.execute("SELECT count(*) FROM packet_seals WHERE packet_id = %s", (packet_id,)).fetchone()
+    return int(r[0]) if r else 0
+
+
+def seal_view(conn: Any, s: Mapping[str, Any] | None) -> dict[str, Any] | None:
+    """The newest seal row, broken or not, with who sealed it and the data book filed with it."""
+    if s is None:
+        return None
+    people = People(conn, [s["sealed_by"], s["broken_by"]])
+    book: dict[str, Any] | None = None
+    aid = s.get("databook_asset_id")
+    if aid is not None:
+        a = one(conn, "SELECT id, size, original_name FROM assets WHERE id = %s", (aid,))
+        book = {
+            "asset_id": str(aid),
+            "url": f"/_blob/{aid}",
+            "download_url": f"/_blob/{aid}?download=1",
+            "pages": s.get("databook_pages"),
+            "size": None if a is None else a["size"],
+            "file_name": None if a is None else a["original_name"],
+        }
+    wos = s.get("open_wos")
+    return {
+        "id": s["id"],
+        "sealed_at": s["sealed_at"],
+        "sealed_by": people.ref(s["sealed_by"]),
+        "key_id": s["key_id"],
+        "signed": s["signature_id"] is not None,
+        "broken_at": s["broken_at"],
+        "broken_by": people.ref(s["broken_by"]),
+        "broken_reason": s["broken_reason"],
+        "open_wos": None if wos is None else list(wos),
+        "databook": book,
+    }
+
+
 def build(
     conn: Any,
     row: Mapping[str, Any],
@@ -787,16 +876,15 @@ def build(
         out["qc_last"] = out["qc_history"][-1] if out["qc_history"] else None
         out["change_orders"] = [_co(c, ch, people) for c in ch.change_orders]
         out["docs"] = [_doc(d, people, seal=seal) for d in ch.docs]
-    out["seal"] = (
-        None
-        if ch.seal is None
-        else {"sealed_at": ch.seal["sealed_at"], "key_id": ch.seal["key_id"], "broken_at": ch.seal["broken_at"]}
-    )
+    if not seal:
+        out["seal"] = seal_view(conn, ch.seal)
+        out["seals"] = seal_count(conn, row["id"]) if ch.seal is not None else 0
     out["checklist_items"] = checklists.checklist_for(weld, row["drawing"], row["rev"], row["wps"], ch.templates)
     out["qc_items"] = qc_items_view(row, ch)
     out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"], row["visual"])
     if not seal:
         out["visual_state"] = visual_state(conn, row)
+        out["vclose_check"] = vclose_check(row, out["visual_state"])
         if include_children:
             from . import shop  # shop builds on this module
 

server/packetstation/domain/pipeline.py

@@ -14,12 +14,16 @@ Child-row endpoints build on the same pieces: `check_row_version`, `next_counter
 
 from __future__ import annotations
 
+import contextlib
 import hashlib
 import hmac
 import logging
+import os
+import tempfile
 from collections.abc import Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any, Literal
 
 from fastapi import Request
@@ -45,6 +49,16 @@ MSG_STALE = "Someone else already took this step."
 MSG_VERSION = "The packet changed since you opened it. Reload and try again."
 MSG_ROW = "Someone else changed this line. Reload and try again."
 MSG_FIELD = "Someone else changed this since you opened it."
+MSG_DATABOOK = "The data book could not be built: {reason}. Nothing was filed."
+
+
+def open_wos_text(wos: Sequence[str]) -> str:
+    """The FILE confirm question: "VISUAL still shows 2 work orders open on this packet: L489052/1, L489052/2. File
+    anyway?" """
+    n = len(wos)
+    what = "1 work order" if n == 1 else f"{n} work orders"
+    return f"VISUAL still shows {what} open on this packet: {', '.join(wos)}. File anyway?"
+
 
 # Codes that answer 400 (the UI asks for input and retries); every other refusal answers 403.
 ASK_CODES = frozenset({"it_reason_required", "reason_required"})
@@ -143,6 +157,17 @@ class Write:
     def rule(self) -> policy.Rule:
         return policy.RULES[self.action]
 
+    @property
+    def settings(self) -> Any:
+        return _settings(self)
+
+    @property
+    def actor_ref(self) -> Any:
+        """The signed-in person as the PDF builders' WhoRef: id, display name and employee number."""
+        from . import prints
+
+        return prints.who_of(self.principal)
+
     def log(self, action_label: str, note: str | None = None) -> None:
         """Override the packet_log label (and note) for this write."""
         self.label = action_label
@@ -323,6 +348,11 @@ def packet_write(
             bypass = bypass or dd.admin_bypass
         if if_match is not None and int(if_match) != int(row["version"]):
             raise ApiError(409, "version_conflict", MSG_VERSION, extra={"current": current_of(row)})
+        if action is A.FILE and extra.get("confirm_open_wos") is not True:
+            # Before the signature: a cancelled confirm costs no attempt and writes nothing.
+            wos = packets.packet_open_wos(conn, row)
+            if wos:
+                raise ApiError(409, "open_work_orders", open_wos_text(wos), extra={"work_orders": wos})
         w = Write(
             conn=conn,
             principal=p,
@@ -770,14 +800,7 @@ def apply_transition(w: Write) -> None:
     _set_packet(w, "status = %s::packet_status, status_at = now(), dept_code = %s", (to_status, to_dept))
 
     if a is A.FILE:
-        key = _seal_key(w)
-        row = packets.load_row(w.conn, w.id)
-        assert row is not None
-        content = packets.seal_content(w.conn, row)
-        w.conn.execute(
-            "INSERT INTO packet_seals (packet_id, hash, key_id, sealed_by, signature_id) VALUES (%s, %s, %s, %s, %s)",
-            (w.id, seal_hash(key, content), seal_key_id(key), w.user_id, w.signature.id if w.signature else None),
-        )
+        _file(w)
     elif a is A.ADMIN_REOPEN:
         w.conn.execute(
             "UPDATE packet_seals SET broken_at = now(), broken_by = %s, broken_reason = %s"
@@ -807,6 +830,84 @@ def apply_transition(w: Write) -> None:
         w.note = reason or _note(w)
 
 
+def _file(w: Write) -> None:
+    """FILE: the data book date, the seal, and the data book built and stored on the seal, all in this transaction.
+
+    A data book that cannot be built answers 500 databook_failed and the whole filing rolls back."""
+    key = _seal_key(w)
+    _set_packet(w, "databook_at = now()")
+    row = packets.load_row(w.conn, w.id)
+    assert row is not None
+    wos = packets.packet_open_wos(w.conn, row) if w.extra.get("confirm_open_wos") is True else []
+    content = packets.seal_content(w.conn, row)
+    digest = seal_hash(key, content)
+    kid = seal_key_id(key)
+    seal_id, sealed_at = w.conn.execute(
+        "INSERT INTO packet_seals (packet_id, hash, key_id, sealed_by, signature_id, open_wos)"
+        " VALUES (%s, %s, %s, %s, %s, %s) RETURNING id, sealed_at",
+        (w.id, digest, kid, w.user_id, w.signature.id if w.signature else None, wos or None),
+    ).fetchone()
+    pages = _file_databook(w, row, {"hash": digest.hex(), "key_id": kid, "sealed_at": sealed_at}, int(seal_id))
+    w.log_also("Data book filed", f"{pages} pages")
+    if wos:
+        w.note = " · ".join(x for x in (_note(w), "Filed with work orders open in VISUAL: " + ", ".join(wos)) if x)
+        w.detail["open_wos"] = wos
+    w.detail["seal_id"] = int(seal_id)
+    w.detail["databook_pages"] = pages
+
+
+def _file_databook(w: Write, row: Mapping[str, Any], seal: dict[str, Any], seal_id: int) -> int:
+    """Build the filed data book, store it as an asset and put it on the seal row. Returns its page count."""
+    from ..pdf import databook
+    from ..pdf.style import PdfError
+    from . import assets
+    from . import prints as prints_dom
+
+    settings = _settings(w)
+    try:
+        built = databook.build(w.conn, settings, w.id, who=w.actor_ref, seal=seal, mode="filed")
+    except PdfError as e:
+        log.warning("data book for packet %s failed: %s", w.id, e)
+        raise ApiError(500, "databook_failed", MSG_DATABOOK.format(reason=_reason_text(e))) from None
+    read = packets.read_packet(w.conn, w.id, None, include_children=False)
+    name = prints_dom.databook_name(read)
+    tmp_dir = Path(settings.assets_dir) / "tmp"
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(prefix="databook-", suffix=".pdf", dir=tmp_dir)
+    with os.fdopen(fd, "wb") as f:
+        f.write(built.bytes)
+    try:
+        stored = assets.store_file(
+            w.conn,
+            settings,
+            Path(tmp_name),
+            mime=assets.PDF,
+            original_name=name,
+            source="databook",
+            source_ref={"packet_id": w.id, "seal_id": seal_id},
+            uploaded_by=w.user_id,
+            packet_id=w.id,
+        )
+    except ApiError as e:
+        log.warning("data book for packet %s was not stored: %s", w.id, e.detail)
+        raise ApiError(500, "databook_failed", MSG_DATABOOK.format(reason=e.detail.rstrip("."))) from None
+    finally:
+        with contextlib.suppress(OSError):
+            Path(tmp_name).unlink()
+    pages = int(built.pages)
+    w.conn.execute(
+        "UPDATE packet_seals SET databook_asset_id = %s, databook_pages = %s WHERE id = %s",
+        (stored["id"], pages, seal_id),
+    )
+    w.detail["databook_asset_id"] = str(stored["id"])
+    return pages
+
+
+def _reason_text(e: BaseException) -> str:
+    text = " ".join(str(e).split()).rstrip(".")
+    return text[:300] or "an unknown error"
+
+
 def verify_seal(conn: Any, settings: Any, packet_id: int) -> dict[str, Any]:
     """The seal history of a packet and whether the current unbroken seal still matches the packet."""
     seals = packets.rows(conn, "SELECT * FROM packet_seals WHERE packet_id = %s ORDER BY id DESC", (packet_id,))
@@ -834,6 +935,9 @@ def verify_seal(conn: Any, settings: Any, packet_id: int) -> dict[str, Any]:
             "broken_at": s["broken_at"],
             "broken_by": people.ref(s["broken_by"]),
             "broken_reason": s["broken_reason"],
+            "open_wos": None if s.get("open_wos") is None else list(s["open_wos"]),
+            "databook_asset_id": None if s.get("databook_asset_id") is None else str(s["databook_asset_id"]),
+            "databook_pages": s.get("databook_pages"),
         }
 
     return {"seal": shape(current) if current else None, "valid": valid, "seals": [shape(s) for s in seals]}

server/packetstation/domain/prints.py

@@ -0,0 +1,182 @@
+"""Printed copies and exports: who the PDF builders stamp, the file names, and the append-only print log.
+
+Every print and every export writes one print_log row and one audit row, in its own transaction, after the bytes
+were built. Prints and exports are not packet changes: they write no packet_log row and do not bump the version.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Literal, cast
+from urllib.parse import quote
+
+from .. import audit
+from ..api import deps
+from . import assets
+from . import packets as packets_dom
+
+if TYPE_CHECKING:
+    from ..pdf.style import WhoRef
+
+PRINTS_LIMIT = 200
+
+# The print_log kinds of the exports (prints use the print kind itself), and the audit kind of each.
+EXPORT_KINDS = {"databook-export": "databook", "traveler": "traveler", "csv": "csv", "itp-pdf": "itp"}
+
+
+def who_of(p: deps.Principal) -> WhoRef:
+    """The signed-in person as the builders' WhoRef: {"id", "display_name", "emp_no"}."""
+    emp = p.user.get("emp_no")
+    return cast(
+        "WhoRef",
+        {
+            "id": str(p.user_id),
+            "display_name": str(p.user.get("display_name") or p.user.get("sam_account") or ""),
+            "emp_no": None if emp is None or str(emp).strip() == "" else str(emp),
+        },
+    )
+
+
+def who_of_user(conn: Any, user_id: Any) -> WhoRef:
+    """The same WhoRef for a user id (the ITP hook gets an actor, not a principal)."""
+    r = conn.execute("SELECT display_name, emp_no FROM users WHERE id = %s", (user_id,)).fetchone()
+    name = str(r[0] or "") if r else ""
+    emp = r[1] if r else None
+    return cast(
+        "WhoRef",
+        {"id": str(user_id), "display_name": name, "emp_no": None if emp is None or str(emp) == "" else str(emp)},
+    )
+
+
+# ---------------------------------------------------------------- file names
+
+
+def job_base(read: Mapping[str, Any]) -> str:
+    """{project}-Lot{lot}: the top project number and the unit's lot, as the old build named its files."""
+    unit = read.get("unit") or {}
+    project = (unit.get("project") or {}).get("number") or ""
+    lot = unit.get("lot") or unit.get("number") or ""
+    return f"{project}-Lot{lot}"
+
+
+def _named(text: str, fallback: str) -> str:
+    return assets.safe_name(text) or fallback
+
+
+def databook_name(read: Mapping[str, Any]) -> str:
+    return _named(f"{job_base(read)}_{read.get('drawing') or ''}_Manufacturing_Data_Book.pdf", "Data_Book.pdf")
+
+
+def traveler_name(read: Mapping[str, Any]) -> str:
+    return _named(f"{job_base(read)}_Job_Traveler.pdf", "Job_Traveler.pdf")
+
+
+def print_name(read: Mapping[str, Any], what: str) -> str:
+    return _named(f"{job_base(read)} {what}.pdf", "Print.pdf")
+
+
+def csv_name(read: Mapping[str, Any]) -> str:
+    return _named(f"{job_base(read)}_{read.get('drawing') or ''}_traceability.csv", "traceability.csv")
+
+
+def itp_name(no: Any) -> str:
+    return _named(f"{no or 'ITP'}.pdf", "ITP.pdf")
+
+
+def disposition(filename: str, download: bool = False) -> str:
+    """Content-Disposition: inline unless a download is asked for. An ASCII filename, plus filename* (RFC 5987) when
+    the name has characters outside ASCII."""
+    kind = "attachment" if download else "inline"
+    plain = filename.encode("ascii", "replace").decode("ascii").replace("?", "_")
+    plain = plain.replace('"', "").replace(";", "_").replace("\\", "_").strip() or "download"
+    out = f'{kind}; filename="{plain}"'
+    if plain != filename:
+        out += f"; filename*=UTF-8''{quote(filename, safe='')}"
+    return out
+
+
+# ---------------------------------------------------------------- the log
+
+
+def record(
+    conn: Any,
+    p: deps.Principal,
+    *,
+    action: Literal["print", "export"],
+    kind: str,
+    what: str,
+    pages: int | None,
+    packet_id: int | None = None,
+    unit_id: int | None = None,
+    itp_id: int | None = None,
+    asset_id: Any = None,
+    stamp: str | None = None,
+    stamp_sig: str | None = None,
+    detail: Mapping[str, Any] | None = None,
+) -> int:
+    """One print_log row and one audit row (action `print` or `export`), in one transaction. The audit detail of an
+    export names its kind as databook, traveler, csv or itp."""
+    audit_detail: dict[str, Any] = {
+        "kind": EXPORT_KINDS.get(kind, kind) if action == "export" else kind,
+        "what": what,
+        "pages": pages,
+        "stamp_sig": stamp_sig,
+    }
+    audit_detail.update({k: v for k, v in (detail or {}).items() if v is not None})
+    if asset_id is not None:
+        audit_detail["asset_id"] = str(asset_id)
+    if itp_id is not None:
+        audit_detail["itp_id"] = itp_id
+    with conn.transaction():
+        new_id = conn.execute(
+            "INSERT INTO print_log (by, by_name, session_id, packet_id, unit_id, itp_id, kind, what, pages, asset_id,"
+            " stamp, stamp_sig, ip) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id",
+            (
+                p.user_id,
+                p.user.get("display_name"),
+                p.session_id,
+                packet_id,
+                unit_id,
+                itp_id,
+                kind,
+                what,
+                pages,
+                asset_id,
+                stamp,
+                stamp_sig,
+                p.ip,
+            ),
+        ).fetchone()[0]
+        audit.append(
+            conn,
+            action=action,
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="itp" if itp_id is not None and packet_id is None else "packet",
+            subject_id=itp_id if itp_id is not None and packet_id is None else packet_id,
+            packet_id=packet_id,
+            detail=audit_detail,
+            ip=p.ip,
+        )
+    return int(new_id)
+
+
+def list_prints(conn: Any, packet_id: int, limit: int = PRINTS_LIMIT) -> list[dict[str, Any]]:
+    """A packet's printed copies and exports, newest first."""
+    return [
+        {
+            "id": r["id"],
+            "at": r["at"],
+            "by": None if r["by"] is None else {"id": str(r["by"]), "display_name": r["by_name"]},
+            "kind": r["kind"],
+            "what": r["what"],
+            "pages": r["pages"],
+        }
+        for r in packets_dom.rows(
+            conn,
+            "SELECT id, at, by, by_name, kind, what, pages FROM print_log WHERE packet_id = %s"
+            " ORDER BY id DESC LIMIT %s",
+            (packet_id, limit),
+        )
+    ]