Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/domain/packets.py

@@ -7,6 +7,7 @@ boards from header columns, v_open_cos and the open short count only.
 
 from __future__ import annotations
 
+import re
 import uuid
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
@@ -20,7 +21,7 @@ from ..api.errors import ApiError
 from . import checklists
 
 HAND_FLAG = "Made by hand - check it and send it on"
-DRAWING_NEEDED_FLAG = "Drawing needed - looking it up in the vault"
+DRAWING_NEEDED_FLAG = "Made from VISUAL - drawing not attached yet"
 DRAWING_STATES_ATTACHED = ("attached", "hand")
 
 # Keys left out of the sealed content: they change without the packet changing (who can do what, the counter,
@@ -37,6 +38,7 @@ SEAL_EXCLUDE = (
     "checklist_items",
     "qc_items",
     "flag",
+    "visual_state",
 )
 
 
@@ -67,19 +69,102 @@ def load_row(conn: Any, packet_id: int, *, lock: bool = False) -> dict[str, Any]
     return one(conn, sql, (packet_id,))
 
 
-def flag_of(status: str, source_kind: str, flags: Iterable[str] | None, drawing_state: str | None = None) -> str | None:
-    """The draft flag shown on cards and the packet page. The old packetFlag(), phase 2 part."""
+# Flags the drawing job owns; a hand drawing or an approved rev clears them.
+DRAWING_FLAGS = ("drawing_needed", "rev_mismatch", "not_released", "not_found", "lookup_failed", "bad_rev")
+# drawing_state values that mean the drawing still needs attention.
+DRAWING_STATES_FLAGGED = ("needed", "rev_mismatch", "not_released", "not_found", "lookup_failed")
+HAND_UPLOAD_FLAGS = ("hand_drawing", "hand_bom")
+
+# Flag kinds in display priority (the first one present gives the text).
+FLAG_PRIORITY = (
+    "hand_drawing",
+    "hand_bom",
+    "rev_mismatch",
+    "not_released",
+    "not_found",
+    "lookup_failed",
+    "bad_rev",
+    "drawing_needed",
+    "no_drawing",
+    "wo_gone",
+    "cancelled",
+)
+FLAG_TEXTS = {
+    "not_found": "Drawing not in M-Files - check the drawing number in VISUAL",
+    "bad_rev": "VISUAL rev is not a number - drawing not attached",
+    "lookup_failed": "M-Files lookup failed",
+    "drawing_needed": DRAWING_NEEDED_FLAG,
+    "no_drawing": "No drawing number in VISUAL - add it there or upload the drawing by hand",
+    "wo_gone": "Work order no longer in VISUAL",
+    "cancelled": "Work order cancelled in VISUAL",
+}
+# drawing_state -> flag kind, for a visual draft whose flags list does not name the problem.
+STATE_KINDS = {
+    "needed": "drawing_needed",
+    "rev_mismatch": "rev_mismatch",
+    "not_released": "not_released",
+    "not_found": "not_found",
+    "lookup_failed": "lookup_failed",
+}
+RX_FLAG_KIND = re.compile(r"^[a-z_]{1,30}$")
+
+
+def _appr_note(visual: Mapping[str, Any] | None) -> str:
+    appr = (visual or {}).get("appr") if isinstance(visual, Mapping) else None
+    if isinstance(appr, Mapping):
+        note = appr.get("note")
+        if isinstance(note, str) and note.strip():
+            return note.strip()
+    return ""
+
+
+def flag_of(
+    status: str,
+    source_kind: str,
+    flags: Iterable[str] | None,
+    drawing_state: str | None = None,
+    visual: Mapping[str, Any] | None = None,
+) -> str | None:
+    """The draft flag shown on cards and the packet page: the old packetFlag(), handFlag() and revMismatch() texts.
+
+    Only drafts carry a flag. A hand draft keeps the phase 2 text. A VISUAL draft shows its most important flag kind.
+    """
     if status != "draft":
         return None
     if source_kind == "hand":
         return HAND_FLAG
-    if "drawing_needed" in (flags or ()):
-        return DRAWING_NEEDED_FLAG
-    return None
+    kinds = list(flags or ())
+    if not kinds and drawing_state in STATE_KINDS:
+        kinds = [STATE_KINDS[drawing_state]]
+    if not kinds:
+        return None
+    dwg, bom = "hand_drawing" in kinds, "hand_bom" in kinds
+    if dwg or bom:
+        what = "Drawing and BOM" if dwg and bom else "Drawing" if dwg else "BOM"
+        return f"{what} uploaded by hand - check it and send it on"
+    note = _appr_note(visual)
+    for kind in FLAG_PRIORITY:
+        if kind not in kinds:
+            continue
+        if kind == "rev_mismatch":
+            return note or "Rev mismatch: VISUAL and M-Files show different revs"
+        if kind == "not_released":
+            return note or "Not released in M-Files yet"
+        if kind in ("lookup_failed", "bad_rev"):
+            # The drawing job writes the whole text ("M-Files lookup failed: ...", "VISUAL rev A is not a number;
+            # M-Files has rev 3 - not attached").
+            return note or FLAG_TEXTS[kind]
+        return FLAG_TEXTS[kind]
+    return "Flagged: " + ", ".join(k.replace("_", " ") for k in kinds)
 
 
 # SQL for the same rule, for the list filter.
-FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR 'drawing_needed' = ANY(p.flags)))"
+FLAG_SQL = "(p.status = 'draft' AND (p.source_kind = 'hand' OR cardinality(p.flags) > 0))"
+# A pending change order from the feed (VISUAL or M-Files): the "Changes to approve" board.
+CO_PENDING_SQL = (
+    "EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'"
+    " AND c.origin IN ('visual', 'drawing'))"
+)
 
 
 def policy_principal_for(conn: Any, p: deps.Principal) -> policy.Principal:
@@ -378,6 +463,57 @@ def _unit(conn: Any, unit_id: int, *, seal: bool) -> dict[str, Any]:
     }
 
 
+def _visual(conn: Any, row: Mapping[str, Any], *, seal: bool) -> dict[str, Any]:
+    """The packet's visual jsonb. On a VISUAL draft (outside seal mode) it gains `staged_rev`: the M-Files copy the
+    drawing job staged for this drawing ({rev, name}), or None. It changes without the packet changing, so the seal
+    leaves it out."""
+    out = dict(row["visual"] or {})
+    if seal or row["source_kind"] != "visual" or row["status"] != "draft":
+        return out
+    out["staged_rev"] = staged_rev(conn, row["drawing"])
+    return out
+
+
+def staged_rev(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    lk = staged_lookup(conn, drawing)
+    if lk is None:
+        return None
+    return {"rev": lk["rev"], "name": lk["name"]}
+
+
+def staged_lookup(conn: Any, drawing: Any) -> dict[str, Any] | None:
+    """The drawing_lookups row for a drawing number when it holds a fetched copy (asset and rev set)."""
+    dwg = str(drawing or "").strip()
+    if not dwg:
+        return None
+    return one(
+        conn,
+        "SELECT * FROM drawing_lookups WHERE dwg = %s AND asset_id IS NOT NULL AND rev IS NOT NULL",
+        (dwg,),
+    )
+
+
+def visual_state(conn: Any, row: Mapping[str, Any]) -> dict[str, Any] | None:
+    """What the feed last observed for a VISUAL packet: leg statuses, ticket hours, the chosen order, and the closed,
+    cancelled and gone times. None for a hand packet."""
+    if row["source_kind"] != "visual":
+        return None
+    vs = one(
+        conn,
+        "SELECT legs, ticket_hours, order_ref, observed_at FROM packet_visual_state WHERE packet_id = %s",
+        (row["id"],),
+    )
+    return {
+        "legs": (vs or {}).get("legs") or [],
+        "ticket_hours": (vs or {}).get("ticket_hours"),
+        "order_ref": (vs or {}).get("order_ref"),
+        "observed_at": (vs or {}).get("observed_at"),
+        "closed_at": row.get("visual_closed_at"),
+        "cancelled_at": row.get("visual_cancelled_at"),
+        "wo_gone_at": row.get("wo_gone_at"),
+    }
+
+
 def build(
     conn: Any,
     row: Mapping[str, Any],
@@ -424,7 +560,7 @@ def build(
         "source": row["source"] or {},
         "drawing_state": row["drawing_state"],
         "flags": list(row["flags"] or []),
-        "visual": row["visual"] or {},
+        "visual": _visual(conn, row, seal=seal),
         "removed": None
         if row["removed_at"] is None
         else {"at": row["removed_at"], "by": people.ref(row["removed_by"]), "reason": row["removed_reason"]},
@@ -451,6 +587,7 @@ def build(
                     "row_version",
                 )
             }
+            | {k: w.get(k) for k in ("dwg", "rev", "partspecs")}
             for w in ch.work_orders
         ]
         out["pages"] = [
@@ -513,7 +650,9 @@ def build(
     )
     out["checklist_items"] = checklists.checklist_for(weld, row["drawing"], row["rev"], row["wps"])
     out["qc_items"] = checklists.qc_items_for(row["drawing"], row["rev"], row["wps"], row["welder_stamp"])
-    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"])
+    out["flag"] = flag_of(row["status"], row["source_kind"], row["flags"], row["drawing_state"], row["visual"])
+    if not seal:
+        out["visual_state"] = visual_state(conn, row)
     if pp is not None:
         allowed, blocked = policy.can(pp, state)
         out["can"] = [str(a) for a in allowed]
@@ -599,8 +738,12 @@ def _co(c: Mapping[str, Any], ch: Children, people: People) -> dict[str, Any]:
         "rejected_at",
         "rejected_reason",
         "row_version",
+        "vis",
+        "waiting",
+        "staged",
+        "target_fp",
     )
-    out = {k: c[k] for k in keys}
+    out = {k: c.get(k) for k in keys}
     out["issued_by"] = people.ref(c["issued_by"])
     out["approved_by"] = people.ref(c["approved_by"])
     out["rejected_by"] = people.ref(c["rejected_by"])
@@ -641,7 +784,10 @@ SELECT p.id, p.version, p.unit_id, u.lot, pr.number || u.lot AS unit_number, pr.
        st.name AS station_name, p.welder_id, wu.display_name AS welder_name, p.welder_stamp,
        p.routed_to::text AS routed_to, p.vclosed_at, p.vclosed_by, vu.display_name AS vclosed_name,
        p.source_kind::text AS source_kind, p.flags, p.drawing_state::text AS drawing_state, p.wh_skipped,
-       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at,
+       p.heat_map, p.removed_at, coalesce(u.due, pr.due) AS due_effective, p.updated_at, p.visual,
+       p.visual_closed_at, p.visual_cancelled_at, p.wo_gone_at,
+       EXISTS (SELECT 1 FROM change_orders c WHERE c.packet_id = p.id AND c.review = 'pending'
+               AND c.origin IN ('visual', 'drawing')) AS co_pending,
        (SELECT w.stage::text FROM packet_wh w WHERE w.packet_id = p.id AND w.closed_at IS NULL) AS wh_stage,
        (SELECT count(*) FROM bom_shorts s WHERE s.packet_id = p.id AND s.closed_at IS NULL) AS open_shorts,
        (SELECT count(*) FROM packet_work_orders wo WHERE wo.packet_id = p.id) AS wo_count,
@@ -697,7 +843,8 @@ def card_state(r: Mapping[str, Any], names: Mapping[str, str]) -> policy.PacketS
 
 
 def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[str, Any]) -> list[dict[str, Any]]:
-    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag, limit, removed."""
+    """The board cards. Filters: status (comma list), dept, unit_id, project_id, q, flag (1|0|hand|hand_upload|
+    <kind>), co (pending), limit, removed."""
     where: list[str] = []
     params: list[Any] = []
     if not filters.get("removed"):
@@ -721,7 +868,24 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
     elif flag in FLAG_OFF:
         where.append("NOT " + FLAG_SQL)
     elif flag == "hand":
-        where.append("(p.status = 'draft' AND p.source_kind = 'hand')")
+        # Made by hand, or a VISUAL draft with a drawing or BOM uploaded by hand.
+        where.append("(p.status = 'draft' AND (p.source_kind = 'hand' OR p.flags && %s::text[]))")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag in ("hand_upload", "hand_uploads"):
+        where.append("(p.status = 'draft' AND p.flags && %s::text[])")
+        params.append(list(HAND_UPLOAD_FLAGS))
+    elif flag and RX_FLAG_KIND.match(flag):
+        # One flag kind on the Flagged board (drafts). A visual draft whose flags are empty but whose drawing state
+        # still names the problem counts too, the same way flag_of reads it.
+        states = [st for st, k in STATE_KINDS.items() if k == flag]
+        where.append(
+            "(p.status = 'draft' AND (%s = ANY(p.flags) OR (cardinality(p.flags) = 0 AND p.source_kind = 'visual'"
+            " AND p.drawing_state::text = ANY(%s))))"
+        )
+        params += [flag, states]
+    co = str(filters.get("co") or "").strip().lower()
+    if co == "pending":
+        where.append(CO_PENDING_SQL)
     for word in str(filters.get("q") or "").split()[:8]:
         pat = _like(word)
         where.append(
@@ -771,7 +935,11 @@ def list_cards(conn: Any, policy_principal: policy.Principal, filters: Mapping[s
                     else {"id": str(r["vclosed_by"]), "display_name": r["vclosed_name"]},
                 },
                 "source_kind": r["source_kind"],
-                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"]),
+                "flag": flag_of(r["status"], r["source_kind"], r["flags"], r["drawing_state"], r["visual"]),
+                "flag_kinds": list(r["flags"] or []),
+                "co_pending": bool(r["co_pending"]),
+                "visual_closed": r["visual_closed_at"] is not None,
+                "drawing_state": r["drawing_state"],
                 "open_cos": [c["co_no"] for c in r["open_co_rows"] or []],
                 "open_shorts": int(r["open_shorts"] or 0),
                 "wh_stage": r["wh_stage"],