Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/domain/assets.py

@@ -18,6 +18,7 @@ from pathlib import Path
 from typing import Any, BinaryIO
 
 from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
 
 from .. import audit
 from ..api.errors import ApiError
@@ -69,7 +70,35 @@ def path_of(settings: Settings, key: str) -> Path:
 
 
 def pdf_pages(path: Path) -> int:
-    """Page count through pypdf; 422 bad_pdf when it does not parse or has no pages."""
+    """Page count through pypdfium2, or pypdf when pypdfium2 is missing or cannot read the file.
+
+    422 bad_pdf when neither parses it or it has no pages.
+    """
+    n = _pages_pdfium(path)
+    if n is None:
+        n = _pages_pypdf(path)
+    if n < 1:
+        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
+    return n
+
+
+def _pages_pdfium(path: Path) -> int | None:
+    try:
+        import pypdfium2
+    except ImportError:
+        return None
+    try:
+        doc = pypdfium2.PdfDocument(str(path))
+    except Exception as e:  # any pdfium failure falls back to pypdf
+        log.info("pypdfium2 could not read the PDF (%s); trying pypdf", type(e).__name__)
+        return None
+    try:
+        return len(doc)
+    finally:
+        doc.close()
+
+
+def _pages_pypdf(path: Path) -> int:
     from pypdf import PdfReader
     from pypdf.errors import PyPdfError
 
@@ -78,8 +107,6 @@ def pdf_pages(path: Path) -> int:
     except (PyPdfError, ValueError, KeyError, TypeError, OSError, IndexError, AttributeError) as e:
         log.info("upload is not a readable PDF: %s", type(e).__name__)
         raise ApiError(422, "bad_pdf", "That PDF could not be read.") from None
-    if n < 1:
-        raise ApiError(422, "bad_pdf", "That PDF has no pages.")
     return n
 
 
@@ -115,6 +142,40 @@ def _fsync_dir(path: Path) -> None:
         os.close(fd)
 
 
+def _existing(conn: Any, sha: bytes, size: int) -> dict[str, Any] | None:
+    """The stored asset with the same bytes (same sha256 and size), oldest first."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _place(settings: Settings, tmp: Path) -> tuple[uuid.UUID, str, Path]:
+    """Rename a fsynced file to ASSETS_DIR/<id[:2]>/<id>, then fsync the directory. Returns (id, key, path)."""
+    aid = uuid.uuid4()
+    key = storage_key(aid)
+    final = path_of(settings, key)
+    final.parent.mkdir(parents=True, exist_ok=True)
+    os.replace(tmp, final)
+    _fsync_dir(final.parent)
+    return aid, key, final
+
+
+def _insert(conn: Any, values: tuple[Any, ...]) -> dict[str, Any]:
+    """Insert one assets row from (id, sha256, size, mime, name, key, source, source_ref, pages, by, packet_id)."""
+    v = list(values)
+    if v[7] is not None and not isinstance(v[7], Jsonb):
+        v[7] = Jsonb(v[7])
+    with conn.cursor(row_factory=dict_row) as cur:
+        row = cur.execute(
+            "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
+            " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
+            v,
+        ).fetchone()
+    return dict(row)
+
+
 def store_upload(
     conn: Any,
     settings: Settings,
@@ -166,29 +227,15 @@ def store_upload(
         if mime is None or size == 0:
             raise ApiError(415, "unsupported_type", "Only PDF, PNG and JPEG files can be stored.")
         sha = digest.digest()
-        with conn.cursor(row_factory=dict_row) as cur:
-            existing = cur.execute(
-                "SELECT * FROM assets WHERE sha256 = %s AND size = %s ORDER BY uploaded_at LIMIT 1", (sha, size)
-            ).fetchone()
+        existing = _existing(conn, sha, size)
         if existing is not None:
-            return public(dict(existing))
+            return public(existing)
         pages = pdf_pages(tmp) if mime == PDF else None
-        aid = uuid.uuid4()
-        key = storage_key(aid)
-        final = path_of(settings, key)
-        final.parent.mkdir(parents=True, exist_ok=True)
-        os.replace(tmp, final)
+        aid, key, placed = _place(settings, tmp)
         keep = True
-        placed = final
-        _fsync_dir(final.parent)
         name = safe_name(original_name)
         with conn.transaction():
-            with conn.cursor(row_factory=dict_row) as cur:
-                row = cur.execute(
-                    "INSERT INTO assets (id, sha256, size, mime, original_name, storage_key, source, source_ref, pages,"
-                    " uploaded_by, packet_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *",
-                    (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id),
-                ).fetchone()
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, user, packet_id))
             audit.append(
                 conn,
                 action="asset.uploaded",
@@ -213,6 +260,69 @@ def store_upload(
                 tmp.unlink()
 
 
+def store_file(
+    conn: Any,
+    settings: Settings,
+    path: Path,
+    *,
+    mime: str = PDF,
+    original_name: Any,
+    source: str,
+    source_ref: Any = None,
+    uploaded_by: Any = None,
+    packet_id: int | None = None,
+) -> dict[str, Any]:
+    """Store a file the server wrote itself (the worker's M-Files download). Returns the full assets row.
+
+    The file is consumed: renamed into ASSETS_DIR when its bytes are new, deleted when the same bytes are already
+    stored (same sha256 and size; that row comes back). Write it under ASSETS_DIR (ASSETS_DIR/tmp) so the rename
+    stays on one file system. It is hashed in chunks, its first bytes must match `mime`, and it is fsynced before
+    the rename. The row carries one extra key, `reused`. No audit row: the caller writes its own.
+    Raises 413 too_large, 415 unsupported_type, 422 bad_pdf.
+    """
+    path = Path(path)
+    limit = int(settings.upload_max_bytes)
+    placed: Path | None = None
+    done = False
+    try:
+        digest = hashlib.sha256()
+        size = 0
+        head = b""
+        with path.open("r+b") as f:
+            while True:
+                chunk = f.read(CHUNK)
+                if not chunk:
+                    break
+                if len(head) < HEAD:
+                    head += chunk[: HEAD - len(head)]
+                size += len(chunk)
+                if size > limit:
+                    raise ApiError(413, "too_large", f"The file is over the {limit // (1024 * 1024)} MB limit.")
+                digest.update(chunk)
+            os.fsync(f.fileno())
+        if size == 0 or sniff(head) != mime:
+            raise ApiError(415, "unsupported_type", f"The file is not {EXT.get(mime, mime)}.")
+        sha = digest.digest()
+        existing = _existing(conn, sha, size)
+        if existing is not None:
+            return {**existing, "reused": True}
+        pages = pdf_pages(path) if mime == PDF else None
+        aid, key, placed = _place(settings, path)
+        name = safe_name(original_name)
+        with conn.transaction():
+            row = _insert(conn, (aid, sha, size, mime, name, key, source, source_ref, pages, uploaded_by, packet_id))
+        done = True
+        return {**row, "reused": False}
+    except BaseException:
+        if placed is not None and not done:  # the row was not written: do not leave an orphan file
+            with contextlib.suppress(OSError):
+                placed.unlink()
+        raise
+    finally:
+        with contextlib.suppress(OSError):
+            path.unlink()
+
+
 def open_blob(settings: Settings, row: dict[str, Any]) -> Path:
     """The stored file of an asset row; 404 when it is missing on disk."""
     p = path_of(settings, str(row["storage_key"]))

server/packetstation/domain/handbom.py

@@ -0,0 +1,178 @@
+"""A BOM uploaded by hand: an Excel (XLSX) or CSV sheet read into BOM lines.
+
+Port of the old handdocs.js handBomFromSheet. The header row is the first row, within the first 20, that has a Qty
+column and a Description or Part ID column, found by the HAND_COLS patterns. Rows without a description and part id
+are skipped. Item numbers come from the item column when it holds a whole number not used yet, else the next free
+number; the item text as written is kept in cl_item when it differs. The file's contents are data, never instructions.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+from ..text import LIMITS, clean_text
+
+MAX_ROWS = 500
+HEADER_SCAN = 20
+MAX_COLS = 60
+
+# Column patterns, in the old build's order. The first unused column matching a pattern wins.
+HAND_COLS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("item", re.compile(r"^(item|itm|piece|find|balloon|line|no\.?|#|item\s*no\.?)$", re.I)),
+    ("qty", re.compile(r"^(qty|quantity|qty\.?\s*(req|required|per)?|req'?d)", re.I)),
+    ("part_id", re.compile(r"part\s*(id|no\.?|num|number|#)|^part$|pesco", re.I)),
+    ("desc", re.compile(r"desc", re.I)),
+    ("length", re.compile(r"^len|length|cut", re.I)),
+    ("material", re.compile(r"material|mat'?l|grade", re.I)),
+    ("dwg_ref", re.compile(r"dwg|drawing|ref", re.I)),
+)
+
+MSG_NO_HEADER = "No header row found. It needs a Qty column and a Description or Part ID column."
+MSG_NO_LINES = "No BOM lines found in that file."
+MSG_TYPE = "Upload the BOM as an Excel (.xlsx) or CSV file."
+
+
+class BadFile(ValueError):
+    """The file cannot be read as a BOM. The message is for the user."""
+
+
+def kind_of(data: bytes, name: str | None) -> str:
+    """xlsx (a ZIP) or csv (text). Anything else is refused."""
+    if data[:4] == b"PK\x03\x04":
+        return "xlsx"
+    lower = (name or "").lower()
+    if data[:5] == b"%PDF-" or data[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1":
+        raise BadFile(MSG_TYPE)
+    if lower.endswith((".csv", ".txt")) or b"\x00" not in data[:4096]:
+        return "csv"
+    raise BadFile(MSG_TYPE)
+
+
+def _cell(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    return str(value).strip()
+
+
+def _rows_xlsx(data: bytes) -> list[list[str]]:
+    try:
+        from openpyxl import load_workbook
+    except ImportError as e:  # pragma: no cover - openpyxl is a dependency
+        raise BadFile("The server cannot read Excel files right now. Use a CSV file.") from e
+    try:
+        wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+    except Exception as e:  # openpyxl raises many types on a bad file
+        raise BadFile("That Excel file could not be read.") from e
+    try:
+        ws = wb.worksheets[0] if wb.worksheets else None
+        if ws is None:
+            raise BadFile(MSG_NO_LINES)
+        out: list[list[str]] = []
+        for row in ws.iter_rows(values_only=True):
+            out.append([_cell(v) for v in row[:MAX_COLS]])
+            if len(out) > MAX_ROWS + HEADER_SCAN:
+                break
+        return out
+    finally:
+        wb.close()
+
+
+def _rows_csv(data: bytes) -> list[list[str]]:
+    try:
+        text = data.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        text = data.decode("cp1252", errors="replace")
+    sample = text[:8192]
+    try:
+        dialect: Any = csv.Sniffer().sniff(sample, delimiters=",;\t|")
+    except csv.Error:
+        dialect = csv.excel
+    out: list[list[str]] = []
+    for row in csv.reader(io.StringIO(text), dialect):
+        out.append([_cell(v) for v in row[:MAX_COLS]])
+        if len(out) > MAX_ROWS + HEADER_SCAN:
+            break
+    return out
+
+
+def find_header(rows: list[list[str]]) -> tuple[int, dict[str, int]]:
+    """The header row index and the column map, or BadFile."""
+    for i, row in enumerate(rows[:HEADER_SCAN]):
+        m: dict[str, int] = {}
+        for key, rx in HAND_COLS:
+            for j, h in enumerate(row):
+                if h and rx.search(h) and j not in m.values():
+                    m[key] = j
+                    break
+        if "qty" in m and ("desc" in m or "part_id" in m):
+            return i, m
+    raise BadFile(MSG_NO_HEADER)
+
+
+def _qty(text: str) -> Decimal:
+    digits = re.sub(r"[^\d.\-]", "", text)
+    try:
+        q = Decimal(digits) if digits else Decimal(0)
+    except InvalidOperation:
+        return Decimal(0)
+    return q if q.is_finite() else Decimal(0)
+
+
+def lines_from_rows(rows: list[list[str]]) -> list[dict[str, Any]]:
+    hi, m = find_header(rows)
+
+    def cell(r: list[str], key: str) -> str:
+        j = m.get(key)
+        return r[j].strip() if j is not None and j < len(r) else ""
+
+    out: list[dict[str, Any]] = []
+    used: set[int] = set()
+    for r in rows[hi + 1 :]:
+        desc, pid, qty = cell(r, "desc"), cell(r, "part_id"), cell(r, "qty")
+        if not desc and not pid:
+            continue  # blank rows, totals, notes
+        raw_item = cell(r, "item")
+        item: int | None = None
+        if re.fullmatch(r"\d{1,5}", raw_item) and 1 <= int(raw_item) <= 100_000 and int(raw_item) not in used:
+            item = int(raw_item)
+        if item is None:
+            item = len(out) + 1
+            while item in used:
+                item += 1
+        used.add(item)
+        cl_item = clean_text(raw_item, LIMITS["part_id"]) or None if raw_item != str(item) else None
+        out.append(
+            {
+                "item": item,
+                "cl_item": cl_item,
+                "qty": _qty(qty),
+                "part_id": clean_text(pid, LIMITS["part_id"]) or None,
+                "description": clean_text(desc, LIMITS["description"]) or None,
+                "length": clean_text(cell(r, "length"), LIMITS["length"]) or None,
+                "material": clean_text(cell(r, "material"), LIMITS["material"]) or None,
+                "dwg_ref": clean_text(cell(r, "dwg_ref"), LIMITS["drawing"]) or None,
+            }
+        )
+        if len(out) >= MAX_ROWS:
+            break
+    if not out:
+        raise BadFile(MSG_NO_LINES)
+    return out
+
+
+def parse(data: bytes, name: str | None) -> list[dict[str, Any]]:
+    """BOM lines from an XLSX or CSV file: [{item, cl_item, qty, part_id, description, length, material, dwg_ref}].
+
+    Raises BadFile with the reason for the user.
+    """
+    if not data:
+        raise BadFile("The file is empty.")
+    kind = kind_of(data, name)
+    rows = _rows_xlsx(data) if kind == "xlsx" else _rows_csv(data)
+    return lines_from_rows(rows)

server/packetstation/domain/packets.py

@@ -7,6 +7,7 @@ boards from header columns, v_open_cos and the open short count only.
 
 from __future__ import annotations
 
+import re
 import uuid
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
@@ -20,7 +21,7 @@ from ..api.errors import ApiError
 from . import checklists
 
 HAND_FLAG = "Made by hand - check it and send it on"
-DRAWING_NEEDED_FLAG = "Drawing needed - looking it up in the vault"
+DRAWING_NEEDED_FLAG = "Made from VISUAL - drawing not attached yet"
 DRAWING_STATES_ATTACHED = ("attached", "hand")
 
 # Keys left out of the sealed content: they change without the packet changing (who can do what, the counter,
@@ -37,6 +38,7 @@ SEAL_EXCLUDE = (
     "checklist_items",
     "qc_items",
     "flag",
+    "visual_state",
 )
 
 
@@ -67,19 +69,102 @@ def load_row(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any]
     return one(conn, sql, (packet_id,))
 
 
-def flag_of(status: str, source_kind: str, flags: Iterable[str] | None, drawing_state: str | None = None) -> str | None:
-    """The draft flag shown on cards and the packet page. The old packetFlag(), phase 2 part."""
+# Flags the drawing job owns; a hand drawing or an approved rev clears them.
+DRAWING_FLAGS = ("drawing_needed", "rev_mismatch", "not_released", "not_found", "lookup_failed", "bad_rev")
+# drawing_state values that mean the drawing still needs attention.
+DRAWING_STATES_FLAGGED = ("needed", "rev_mismatch", "not_released", "not_found", "lookup_failed")
+HAND_UPLOAD_FLAGS = ("hand_drawing", "hand_bom")
+
+# Flag kinds in display priority (the first one present gives the text).
+FLAG_PRIORITY = (
+    "hand_drawing",
+    "hand_bom",
+    "rev_mismatch",
+    "not_released",
+    "not_found",
+    "lookup_failed",
+    "bad_rev",
+    "drawing_needed",
+    "no_drawing",
+    "wo_gone",
+    "cancelled",
+)
+FLAG_TEXTS = {
+    "not_found": "Drawing not in M-Files - check the drawing number in VISUAL",
+    "bad_rev": "VISUAL rev is not a number - drawing not attached",
+    "lookup_failed": "M-Files lookup failed",
+    "drawing_needed": DRAWING_NEEDED_FLAG,
+    "no_drawing": "No drawing number in VISUAL - add it there or upload the drawing by hand",
+    "wo_gone": "Work order no longer in VISUAL",
+    "cancelled": "Work order cancelled in VISUAL",
+}
+# drawing_state -> flag kind, for a visual draft whose flags list does not name the problem.
+STATE_KINDS = {
+    "needed": "drawing_needed",
+    "rev_mismatch": "rev_mismatch",
+    "not_released": "not_released",
+    "not_found": "not_found",
+    "lookup_failed": "lookup_failed",
+}
+RX_FLAG_KIND = re.compile(r"^[a-z_]{1,30}$")
+
+
+def _appr_note(visual: Mapping[str, Any] | None) -> str:
+    appr = (visual or {}).get("appr") if isinstance(visual, Mapping) else None
+    if isinstance(appr, Mapping):
+        note = appr.get("note")
+        if isinstance(note, str) and note.strip():
+            return note.strip()
+    return ""
+
+
+def flag_of(
+    status: str,
+    source_kind: str,
+    flags: Iterable[str] | None,
+    drawing_state: str | None = None,
+    visual: Mapping[str, Any] | None = None,
+) -> str | None:
+    """The draft flag shown on cards and the packet page: the old packetFlag(), handFlag() and revMismatch() texts.
+
+    Only drafts carry a flag. A hand draft keeps the phase 2 text. A VISUAL draft shows its most important flag kind.
+    """
     if status != "draft":
         return None
     if source_kind == "hand":
         return HAND_FLAG
-    if "drawing_needed" in (flags or ()):
-        return DRAWING_NEEDED_FLAG
-    return None
+    kinds = list(flags or ())
+    if not kinds and drawing_state in STATE_KINDS:
+        kinds = [STATE_KINDS[drawing_state]]
+    if not kinds:
+        return None
+    dwg, bom = "hand_drawing" in kinds, "hand_bom" in kinds
+    if dwg or bom:
+        what = "Drawing and BOM" if dwg and bom else "Drawing" if dwg else "BOM"
+        return f"{what} uploaded by hand - check it and send it on"
+    note = _appr_note(visual)
+    for kind in FLAG_PRIORITY:
+        if kind not in kinds:
+            continue
+        if kind == "rev_mismatch":
+            return note or "Rev mismatch: VISUAL and M-Files show different revs"
+        if kind == "not_released":
+            return note or "Not released in M-Files yet"
+        if kind in ("lookup_failed", "bad_rev"):
+            # The drawing job writes the whole text ("M-Files lookup failed: ...", "VISUAL rev A is not a number;
+            # M-Files has rev 3 - not attached").
+            return note or FLAG_TEXTS[kind]
+        return FLAG_TEXTS[kind]
+    return "Flagged: " + ", ".join(k.replace("_", " ") for k in kinds)
 
 
 # SQL for the same rule, for the list filter.
-FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR 'drawing_needed' = ANY(p.flags)))"
+FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR cardinality(p.flags) > 0))"
+# A pending change order from the feed (VISUAL or M-Files): the "Changes to approve" board.
+CO_PENDING_SQL = (
+    "EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'"
+    " AND c.origin IN ('visual', 'drawing'))"
+)
 
 
 def policy_principal_for(conn: Any, p: deps.Principal) -> policy.Principal:
@@ -378,6 +463,57 @@ def _unit(conn: Any, unit_id: int, *, seal: bool) -> dict[str, Any]:
     }
 
 
+def _visual(conn: Any, row: Mapping[str, Any], *, seal: bool) -> dict[str, Any]:
+    """The packet's visual jsonb. On a VISUAL draft (outside seal mode) it gains `staged_rev`: the M-Files copy the
+    drawing job staged for this drawing ({rev, name}), or None. It changes without the packet changing, so the seal
+    leaves it out."""
+    out = dict(row["visual"] or {})
+    if seal or row["source_kind"] != "visual" or row["status"] != "draft":
+        return out
+    out["staged_rev"] = staged_rev(conn, row["drawing"])
+    return out
+
+
+def staged_rev(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    lk = staged_lookup(conn, drawing)
+    if lk is None:
+        return None
+    return {"rev": lk["rev"], "name": lk["name"]}
+
+
+def staged_lookup(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    """The drawing_lookups row for a drawing number when it holds a fetched copy (asset and rev set)."""
+    dwg = str(drawing or "").strip()
+    if not dwg:
+        return None
+    return one(
+        conn,
+        "SELECT * FROM drawing_lookups WHERE dwg = %s AND asset_id IS NOT NULL AND rev IS NOT NULL",
+        (dwg,),
+    )
+
+
+def visual_state(conn: Any, row: Mapping[str, Any]) -> dict[str, Any] | None:
+    """What the feed last observed for a VISUAL packet: leg statuses, ticket hours, the chosen order, and the closed,
+    cancelled and gone times. None for a hand packet."""
+    if row["source_kind"] != "visual":
+        return None
+    vs = one(
+        conn,
+        "SELECT legs, ticket_hours, order_ref, observed_at FROM packet_visual_state WHERE packet_id = %s",
+        (row["id"],),
+    )
+    return {
+        "legs": (vs or {}).get("legs") or [],
+        "ticket_hours": (vs or {}).get("ticket_hours"),
+        "order_ref": (vs or {}).get("order_ref"),
+        "observed_at": (vs or {}).get("observed_at"),
+        "closed_at": row.get("visual_closed_at"),
+        "cancelled_at": row.get("visual_cancelled_at"),
+        "wo_gone_at": row.get("wo_gone_at"),
+    }
+
+
 def build(
     conn: Any,
     row: Mapping[str, Any],
@@ -424,7 +560,7 @@ def build(
         "source": row["source"] or {},
         "drawing_state": row["drawing_state"],
         "flags": list(row["flags"] or []),
-        "visual": row["visual"] or {},
+        "visual": _visual(conn, row, seal=seal),
         "removed": None
         if row["removed_at"] is None
         else {"at": row["removed_at"], "by": people.ref(row["removed_by"]), "reason": row["removed_reason"]},
@@ -451,6 +587,7 @@ def build(
                     "row_version",
                 )
             }
+            | {k: w.get(k) for k in ("dwg", "rev", "partspecs")}
             for w in ch.work_orders
         ]
         out["pages"] = [
@@ -513,7 +650,9 @@ def build(
     )
     out["checklist_items"] = checklists.checklist_for(weld, row["drawing"], row["rev"], row["wps"])
     out["qc_items"] = checklists.qc_items_for(row["drawing"], row["rev"], row["wps"], row["welder_stamp"])
-    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"])
+    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"], row["visual"])
+    if not seal:
+        out["visual_state"] = visual_state(conn, row)
     if pp is not None:
         allowed, blocked = policy.can(pp, state)
         out["can"] = [str(a) for a in allowed]
@@ -599,8 +738,12 @@ def _co(c: Mapping[str, Any], ch: Children, people: People) -> dict[str, Any]:
         "rejected_at",
         "rejected_reason",
         "row_version",
+        "vis",
+        "waiting",
+        "staged",
+        "target_fp",
     )
-    out = {k: c[k] for k in keys}
+    out = {k: c.get(k) for k in keys}
     out["issued_by"] = people.ref(c["issued_by"])
     out["approved_by"] = people.ref(c["approved_by"])
     out["rejected_by"] = people.ref(c["rejected_by"])
@@ -641,7 +784,10 @@ SELECT p.id, p.version, p.unit_id, u.lot, pr.number || u.lot AS unit_number, pr.
        st.name AS station_name, p.welder_id, wu.display_name AS welder_name, p.welder_stamp,
        p.routed_to::text AS routed_to, p.vclosed_at, p.vclosed_by, vu.display_name AS vclosed_name,
        p.source_kind::text AS source_kind, p.flags, p.drawing_state::text AS drawing_state, p.wh_skipped,
-       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at,
+       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at, p.visual,
+       p.visual_closed_at, p.visual_cancelled_at, p.wo_gone_at,
+       EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'
+               AND c.origin IN ('visual', 'drawing')) AS co_pending,
        (SELECT w.stage::text FROM packet_wh w WHERE w.packet_id = p.id AND w.closed_at IS NULL) AS wh_stage,
        (SELECT count(*) FROM bom_shorts s WHERE s.packet_id = p.id AND s.closed_at IS NULL) AS open_shorts,
        (SELECT count(*) FROM packet_work_orders wo WHERE wo.packet_id = p.id) AS wo_count,
@@ -697,7 +843,8 @@ def card_state(r: Mapping[str, Any], names: Mapping[str, str]) -> policy.PacketS
 
 
 def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[str, Any]) -> list[dict[str, Any]]:
-    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag, limit, removed."""
+    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag (1|0|hand|hand_upload|
+    <kind>), co (pending), limit, removed."""
     where: list[str] = []
     params: list[Any] = []
     if not filters.get("removed"):
@@ -721,7 +868,24 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
     elif flag in FLAG_OFF:
         where.append("NOT " + FLAG_SQL)
     elif flag == "hand":
-        where.append("(p.status = 'draft' AND p.source_kind = 'hand')")
+        # Made by hand, or a VISUAL draft with a drawing or BOM uploaded by hand.
+        where.append("(p.status = 'draft' AND (p.source_kind = 'hand' OR p.flags && %s::text[]))")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag in ("hand_upload", "hand_uploads"):
+        where.append("(p.status = 'draft' AND p.flags && %s::text[])")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag and RX_FLAG_KIND.match(flag):
+        # One flag kind on the Flagged board (drafts). A visual draft whose flags are empty but whose drawing state
+        # still names the problem counts too, the same way flag_of reads it.
+        states = [st for st, k in STATE_KINDS.items() if k == flag]
+        where.append(
+            "(p.status = 'draft' AND (%s = ANY(p.flags) OR (cardinality(p.flags) = 0 AND p.source_kind = 'visual'"
+            " AND p.drawing_state::text = ANY(%s))))"
+        )
+        params += [flag, states]
+    co = str(filters.get("co") or "").strip().lower()
+    if co == "pending":
+        where.append(CO_PENDING_SQL)
     for word in str(filters.get("q") or "").split()[:8]:
         pat = _like(word)
         where.append(
@@ -771,7 +935,11 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
                     else {"id": str(r["vclosed_by"]), "display_name": r["vclosed_name"]},
                 },
                 "source_kind": r["source_kind"],
-                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"]),
+                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"], r["visual"]),
+                "flag_kinds": list(r["flags"] or []),
+                "co_pending": bool(r["co_pending"]),
+                "visual_closed": r["visual_closed_at"] is not None,
+                "drawing_state": r["drawing_state"],
                 "open_cos": [c["co_no"] for c in r["open_co_rows"] or []],
                 "open_shorts": int(r["open_shorts"] or 0),
                 "wh_stage": r["wh_stage"],

server/packetstation/domain/projects.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import re
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
-from datetime import date
+from datetime import UTC, date, datetime
 from typing import Any
 
 from psycopg import errors as pgerr
@@ -28,8 +28,11 @@ MAX_UNITS_AT_ONCE = 200
 MAX_WORDS = 8
 LIST_LIMIT = 500
 
-PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes")
-UNIT_FIELDS = ("name", "due", "status", "notes")
+PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+UNIT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+# Order-derived fields: the value in use, where it came from (visual | hand) and what VISUAL last said.
+ORDER_FIELDS = ("customer", "po", "order_date", "ship_date")
+RX_ORDER_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ./_-]{0,39}$")
 
 _DIGITS = re.compile(r"^[0-9]+$")
 _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
@@ -159,6 +162,28 @@ def _removed(r: Mapping[str, Any]) -> dict[str, Any] | None:
 # ---------------------------------------------------------------- shapes
 
 
+def _order_out(r: Mapping[str, Any]) -> dict[str, Any]:
+    """The order-derived columns with their provenance, and the order pin. Shared by projects and units."""
+    pin_by = r.get("order_pin_by")
+    return {
+        "customer_src": r.get("customer_src"),
+        "customer_visual": r.get("customer_visual"),
+        "po_src": r.get("po_src"),
+        "po_visual": r.get("po_visual"),
+        "order_date": r.get("order_date"),
+        "order_date_src": r.get("order_date_src"),
+        "order_date_visual": r.get("order_date_visual"),
+        "ship_date": r.get("ship_date"),
+        "ship_date_src": r.get("ship_date_src"),
+        "ship_date_visual": r.get("ship_date_visual"),
+        "order_ref": r.get("order_ref"),
+        "order_pin": r.get("order_pin"),
+        "order_pin_by": str(pin_by) if pin_by else None,
+        "order_pin_at": r.get("order_pin_at"),
+        "order_pin_reason": r.get("order_pin_reason"),
+    }
+
+
 def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
     return {
         "id": r["id"],
@@ -171,6 +196,7 @@ def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "notes": r["notes"],
         "source": r["source"],
         "needs_details": r["needs_details"],
+        **_order_out(r),
         "units": int(r.get("unit_count") or 0),
         "packets": _counts(r),
         "version": r["version"],
@@ -193,6 +219,9 @@ def unit_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "status": status,
         "status_effective": "closed" if r["project_status"] == "closed" else status,
         "notes": r["notes"],
+        "customer": r.get("customer"),
+        "po": r.get("po"),
+        **_order_out(r),
         "packets": _counts(r),
         "version": r["version"],
         "created_at": r["created_at"],
@@ -322,6 +351,7 @@ def project_detail(conn: Any, project_id: int) -> dict[str, Any]:
     out["unit_count"] = len(rows)
     out["aliases"] = aliases_of(conn, project_id=project_id)
     out["next_lot"] = next_lot(conn, project_id, out["number"])
+    out["orders"] = cached_orders(conn, out["number"], _project_bases(conn, project_id))
     return out
 
 
@@ -370,6 +400,7 @@ def unit_detail(conn: Any, unit_id: int) -> dict[str, Any]:
         raise ApiError(404, "not_found", "No such unit.")
     out["project"] = {k: p[k] for k in ("id", "number", "name", "customer", "po", "due", "status", "version")}
     out["aliases"] = aliases_of(conn, unit_id=unit_id)
+    out["orders"] = cached_orders(conn, p["number"], _unit_bases(conn, unit_id))
     return out
 
 
@@ -479,6 +510,13 @@ def _norm_field(key: str, value: Any, *, unit: bool) -> Any:
         return clean_opt(value, LIMITS["po"])
     if key == "notes":
         return clean_opt(value, LIMITS["notes"])
+    if key == "order_pin":
+        pin = clean_opt(value, LIMITS["order_pin"] + 20)
+        if pin is not None and not RX_ORDER_ID.match(pin):
+            raise ApiError(422, "invalid_order", "Give a VISUAL customer order ID, or nothing to clear the pin.")
+        return pin
+    if key == "order_pin_reason":
+        return clean_opt(value, LIMITS["reason"])
     raise ApiError(422, "invalid_field", f"{key} cannot be changed here.")
 
 
@@ -515,6 +553,100 @@ def _change_detail(row: Mapping[str, Any], changes: Mapping[str, Any]) -> dict[s
     return out
 
 
+def _columns(row: Mapping[str, Any], changes: Mapping[str, Any], who: Who) -> dict[str, Any]:
+    """The columns a field edit writes: a hand customer or po marks its source, a pin records who and when."""
+    cols = dict(changes)
+    for k in ("customer", "po"):
+        if k in cols:
+            cols[f"{k}_src"] = "hand" if cols[k] else None
+    if "order_pin" in cols:
+        if cols["order_pin"] is None:
+            cols.update(order_pin_by=None, order_pin_at=None, order_pin_reason=None)
+        else:
+            cols["order_pin_by"] = who.user_id
+            cols["order_pin_at"] = datetime.now(UTC)
+    return cols
+
+
+def _check_pin(
+    conn: Any, row: Mapping[str, Any], changes: Mapping[str, Any], *, project_number: str, bases: list[str]
+) -> None:
+    """A new pin needs a reason and, when VISUAL's orders are cached for this project, must name one of them."""
+    if "order_pin" not in changes and "order_pin_reason" in changes and row.get("order_pin") is None:
+        raise ApiError(422, "invalid_field", "There is no order pin to give a reason for.")
+    pin = changes.get("order_pin")
+    if pin is None:
+        return
+    if not changes.get("order_pin_reason"):
+        raise ApiError(400, "reason_required", "Say why this order is pinned.")
+    known = [o["cust_order_id"] for o in cached_orders(conn, project_number, bases)]
+    if known and pin.upper() not in {k.upper() for k in known}:
+        raise ApiError(
+            422,
+            "unknown_order",
+            f"Order {pin} is not one of the VISUAL orders for {project_number}.",
+            extra={"orders": known},
+        )
+
+
+def _project_bases(conn: Any, project_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " JOIN units u ON u.id = k.unit_id WHERE u.project_id = %s ORDER BY 1",
+            (project_id,),
+        )
+    ]
+
+
+def _unit_bases(conn: Any, unit_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " WHERE k.unit_id = %s ORDER BY 1",
+            (unit_id,),
+        )
+    ]
+
+
+def cached_orders(conn: Any, project_number: str, bases: Iterable[str]) -> list[dict[str, Any]]:
+    """Every cached VISUAL order of a project: the ones linked to the project number, then the ones linked to its
+    work order bases. Each once, with `via` (project | base) and the link's lines and top jobs."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT o.*, l.kind AS via, l.key, l.rank, l.lines, l.top_jobs FROM visual_order_links l"
+            " JOIN visual_orders o ON o.cust_order_id = l.cust_order_id"
+            " WHERE (l.kind = 'project' AND upper(l.key) = upper(%s)) OR (l.kind = 'base' AND l.key = ANY(%s))"
+            " ORDER BY CASE l.kind WHEN 'project' THEN 0 ELSE 1 END, l.rank, l.key, o.cust_order_id",
+            (project_number, list(bases)),
+        ).fetchall()
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for r in rows:
+        if r["cust_order_id"] in seen:
+            continue
+        seen.add(r["cust_order_id"])
+        out.append(
+            {
+                "cust_order_id": r["cust_order_id"],
+                "status": r["status"],
+                "customer_id": r["customer_id"],
+                "customer_name": r["customer_name"],
+                "customer_po_ref": r["customer_po_ref"],
+                "order_date": r["order_date"],
+                "desired_ship_date": r["desired_ship_date"],
+                "promise_date": r["promise_date"],
+                "via": r["via"],
+                "key": r["key"],
+                "lines": list(r["lines"] or []),
+                "top_jobs": list(r["top_jobs"] or []),
+            }
+        )
+    return out
+
+
 # ---------------------------------------------------------------- writes: projects
 
 
@@ -567,11 +699,8 @@ def update_project(
         row = _lock_project(conn, project_id)
         changes = field_cas(row, values, base, PROJECT_FIELDS, unit=False)
         if changes:
-            cols = dict(changes)
-            if "customer" in cols:
-                cols["customer_src"] = "hand" if cols["customer"] else None
-            if "po" in cols:
-                cols["po_src"] = "hand" if cols["po"] else None
+            _check_pin(conn, row, changes, project_number=row["number"], bases=_project_bases(conn, project_id))
+            cols = _columns(row, changes, who)
             sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE projects SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
@@ -800,10 +929,12 @@ def update_unit(
         row, project = _lock_unit(conn, unit_id)
         changes = field_cas(row, values, base, UNIT_FIELDS, unit=True)
         if changes:
-            sets = ", ".join(f"{k} = %s" for k in changes)
+            _check_pin(conn, row, changes, project_number=project["number"], bases=_unit_bases(conn, unit_id))
+            cols = _columns(row, changes, who)
+            sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE units SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
-                [*changes.values(), unit_id],
+                [*cols.values(), unit_id],
             )
             _audit(
                 conn,