Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/domain/projects.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import re
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
-from datetime import date
+from datetime import UTC, date, datetime
 from typing import Any
 
 from psycopg import errors as pgerr
@@ -28,8 +28,11 @@ MAX_UNITS_AT_ONCE = 200
 MAX_WORDS = 8
 LIST_LIMIT = 500
 
-PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes")
-UNIT_FIELDS = ("name", "due", "status", "notes")
+PROJECT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+UNIT_FIELDS = ("name", "customer", "po", "due", "status", "notes", "order_pin", "order_pin_reason")
+# Order-derived fields: the value in use, where it came from (visual | hand) and what VISUAL last said.
+ORDER_FIELDS = ("customer", "po", "order_date", "ship_date")
+RX_ORDER_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ./_-]{0,39}$")
 
 _DIGITS = re.compile(r"^[0-9]+$")
 _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
@@ -159,6 +162,28 @@ def _removed(r: Mapping[str, Any]) -> dict[str, Any] | None:
 # ---------------------------------------------------------------- shapes
 
 
+def _order_out(r: Mapping[str, Any]) -> dict[str, Any]:
+    """The order-derived columns with their provenance, and the order pin. Shared by projects and units."""
+    pin_by = r.get("order_pin_by")
+    return {
+        "customer_src": r.get("customer_src"),
+        "customer_visual": r.get("customer_visual"),
+        "po_src": r.get("po_src"),
+        "po_visual": r.get("po_visual"),
+        "order_date": r.get("order_date"),
+        "order_date_src": r.get("order_date_src"),
+        "order_date_visual": r.get("order_date_visual"),
+        "ship_date": r.get("ship_date"),
+        "ship_date_src": r.get("ship_date_src"),
+        "ship_date_visual": r.get("ship_date_visual"),
+        "order_ref": r.get("order_ref"),
+        "order_pin": r.get("order_pin"),
+        "order_pin_by": str(pin_by) if pin_by else None,
+        "order_pin_at": r.get("order_pin_at"),
+        "order_pin_reason": r.get("order_pin_reason"),
+    }
+
+
 def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
     return {
         "id": r["id"],
@@ -171,6 +196,7 @@ def project_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "notes": r["notes"],
         "source": r["source"],
         "needs_details": r["needs_details"],
+        **_order_out(r),
         "units": int(r.get("unit_count") or 0),
         "packets": _counts(r),
         "version": r["version"],
@@ -193,6 +219,9 @@ def unit_out(r: Mapping[str, Any]) -> dict[str, Any]:
         "status": status,
         "status_effective": "closed" if r["project_status"] == "closed" else status,
         "notes": r["notes"],
+        "customer": r.get("customer"),
+        "po": r.get("po"),
+        **_order_out(r),
         "packets": _counts(r),
         "version": r["version"],
         "created_at": r["created_at"],
@@ -322,6 +351,7 @@ def project_detail(conn: Any, project_id: int) -> dict[str, Any]:
     out["unit_count"] = len(rows)
     out["aliases"] = aliases_of(conn, project_id=project_id)
     out["next_lot"] = next_lot(conn, project_id, out["number"])
+    out["orders"] = cached_orders(conn, out["number"], _project_bases(conn, project_id))
     return out
 
 
@@ -370,6 +400,7 @@ def unit_detail(conn: Any, unit_id: int) -> dict[str, Any]:
         raise ApiError(404, "not_found", "No such unit.")
     out["project"] = {k: p[k] for k in ("id", "number", "name", "customer", "po", "due", "status", "version")}
     out["aliases"] = aliases_of(conn, unit_id=unit_id)
+    out["orders"] = cached_orders(conn, p["number"], _unit_bases(conn, unit_id))
     return out
 
 
@@ -479,6 +510,13 @@ def _norm_field(key: str, value: Any, *, unit: bool) -> Any:
         return clean_opt(value, LIMITS["po"])
     if key == "notes":
         return clean_opt(value, LIMITS["notes"])
+    if key == "order_pin":
+        pin = clean_opt(value, LIMITS["order_pin"] + 20)
+        if pin is not None and not RX_ORDER_ID.match(pin):
+            raise ApiError(422, "invalid_order", "Give a VISUAL customer order ID, or nothing to clear the pin.")
+        return pin
+    if key == "order_pin_reason":
+        return clean_opt(value, LIMITS["reason"])
     raise ApiError(422, "invalid_field", f"{key} cannot be changed here.")
 
 
@@ -515,6 +553,100 @@ def _change_detail(row: Mapping[str, Any], changes: Mapping[str, Any]) -> dict[s
     return out
 
 
+def _columns(row: Mapping[str, Any], changes: Mapping[str, Any], who: Who) -> dict[str, Any]:
+    """The columns a field edit writes: a hand customer or po marks its source, a pin records who and when."""
+    cols = dict(changes)
+    for k in ("customer", "po"):
+        if k in cols:
+            cols[f"{k}_src"] = "hand" if cols[k] else None
+    if "order_pin" in cols:
+        if cols["order_pin"] is None:
+            cols.update(order_pin_by=None, order_pin_at=None, order_pin_reason=None)
+        else:
+            cols["order_pin_by"] = who.user_id
+            cols["order_pin_at"] = datetime.now(UTC)
+    return cols
+
+
+def _check_pin(
+    conn: Any, row: Mapping[str, Any], changes: Mapping[str, Any], *, project_number: str, bases: list[str]
+) -> None:
+    """A new pin needs a reason and, when VISUAL's orders are cached for this project, must name one of them."""
+    if "order_pin" not in changes and "order_pin_reason" in changes and row.get("order_pin") is None:
+        raise ApiError(422, "invalid_field", "There is no order pin to give a reason for.")
+    pin = changes.get("order_pin")
+    if pin is None:
+        return
+    if not changes.get("order_pin_reason"):
+        raise ApiError(400, "reason_required", "Say why this order is pinned.")
+    known = [o["cust_order_id"] for o in cached_orders(conn, project_number, bases)]
+    if known and pin.upper() not in {k.upper() for k in known}:
+        raise ApiError(
+            422,
+            "unknown_order",
+            f"Order {pin} is not one of the VISUAL orders for {project_number}.",
+            extra={"orders": known},
+        )
+
+
+def _project_bases(conn: Any, project_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " JOIN units u ON u.id = k.unit_id WHERE u.project_id = %s ORDER BY 1",
+            (project_id,),
+        )
+    ]
+
+
+def _unit_bases(conn: Any, unit_id: int) -> list[str]:
+    return [
+        r[0]
+        for r in conn.execute(
+            "SELECT DISTINCT wo.wo_base FROM packet_work_orders wo JOIN packets k ON k.id = wo.packet_id"
+            " WHERE k.unit_id = %s ORDER BY 1",
+            (unit_id,),
+        )
+    ]
+
+
+def cached_orders(conn: Any, project_number: str, bases: Iterable[str]) -> list[dict[str, Any]]:
+    """Every cached VISUAL order of a project: the ones linked to the project number, then the ones linked to its
+    work order bases. Each once, with `via` (project | base) and the link's lines and top jobs."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT o.*, l.kind AS via, l.key, l.rank, l.lines, l.top_jobs FROM visual_order_links l"
+            " JOIN visual_orders o ON o.cust_order_id = l.cust_order_id"
+            " WHERE (l.kind = 'project' AND upper(l.key) = upper(%s)) OR (l.kind = 'base' AND l.key = ANY(%s))"
+            " ORDER BY CASE l.kind WHEN 'project' THEN 0 ELSE 1 END, l.rank, l.key, o.cust_order_id",
+            (project_number, list(bases)),
+        ).fetchall()
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for r in rows:
+        if r["cust_order_id"] in seen:
+            continue
+        seen.add(r["cust_order_id"])
+        out.append(
+            {
+                "cust_order_id": r["cust_order_id"],
+                "status": r["status"],
+                "customer_id": r["customer_id"],
+                "customer_name": r["customer_name"],
+                "customer_po_ref": r["customer_po_ref"],
+                "order_date": r["order_date"],
+                "desired_ship_date": r["desired_ship_date"],
+                "promise_date": r["promise_date"],
+                "via": r["via"],
+                "key": r["key"],
+                "lines": list(r["lines"] or []),
+                "top_jobs": list(r["top_jobs"] or []),
+            }
+        )
+    return out
+
+
 # ---------------------------------------------------------------- writes: projects
 
 
@@ -567,11 +699,8 @@ def update_project(
         row = _lock_project(conn, project_id)
         changes = field_cas(row, values, base, PROJECT_FIELDS, unit=False)
         if changes:
-            cols = dict(changes)
-            if "customer" in cols:
-                cols["customer_src"] = "hand" if cols["customer"] else None
-            if "po" in cols:
-                cols["po_src"] = "hand" if cols["po"] else None
+            _check_pin(conn, row, changes, project_number=row["number"], bases=_project_bases(conn, project_id))
+            cols = _columns(row, changes, who)
             sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE projects SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
@@ -800,10 +929,12 @@ def update_unit(
         row, project = _lock_unit(conn, unit_id)
         changes = field_cas(row, values, base, UNIT_FIELDS, unit=True)
         if changes:
-            sets = ", ".join(f"{k} = %s" for k in changes)
+            _check_pin(conn, row, changes, project_number=project["number"], bases=_unit_bases(conn, unit_id))
+            cols = _columns(row, changes, who)
+            sets = ", ".join(f"{k} = %s" for k in cols)
             conn.execute(
                 f"UPDATE units SET {sets}, version = version + 1, updated_at = now() WHERE id = %s",  # noqa: S608
-                [*changes.values(), unit_id],
+                [*cols.values(), unit_id],
             )
             _audit(
                 conn,