Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource map, job tables, order-derived columns with provenance; work order key includes the operation sequence). The real worker on APScheduler with settings re-read each fire, pause, run now, advisory locks, job_runs and a heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache, suspect-batch guard, the apply in one transaction: drafts, change orders, retire-moved, work order gone, closed and cancelled observed, projects and units from order links, auto-release) and the fingerprint. The drawing-lookup job (queue, budget, download on version change, sha256 into assets, decision table, staging for rev approval, drawing-origin change orders). Clerical From VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin Feed settings and resource map. Synthetic fixture generator and a fixture-driven worker in CI with a Playwright scenario. Docs updated. Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI stack, 76 frontend tests, doctor at 0004; one feed and one drawing run against VISUAL DEV and the vault from the developer machine (counts in the changelog). Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 14:45:38 -06:00
parent 16950055e3
commit 591fe90076
103 changed files with 16063 additions and 151 deletions
@@ -1392,6 +1392,191 @@ def issue_co(
    return _read(d, pp, packet_id)


+FEED_ORIGINS = ("visual", "drawing")
+MSG_CO_NOT_STAGED = "The new drawing has not been fetched from M-Files yet."
+
+
+def _vis_text(leg: Any, *names: str) -> str:
+    for n in names:
+        v = leg.get(n) if isinstance(leg, dict) else None
+        if v is not None and v != "":
+            return str(v)
+    return ""
+
+
+def _leg_row(leg: Any) -> dict[str, Any]:
+    """One work order row from a change order's `vis.legs` entry. Accepts a flat leg ({wo|wo_base|base, lot, split,
+    sub, seq, ...}) or one with a nested key ({key: {base, lot, split, sub}, seq, ...})."""
+    if not isinstance(leg, dict):
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    key = leg.get("key") if isinstance(leg.get("key"), dict) else leg
+    base = _vis_text(key, "base", "wo_base", "wo")
+    if not base:
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    seq_raw = leg.get("seq")
+    return {
+        "wo_base": base,
+        "lot": _vis_text(key, "lot"),
+        "split": _vis_text(key, "split"),
+        "sub": _vis_text(key, "sub"),
+        "seq": int(seq_raw) if seq_raw not in (None, "") else None,
+        "qty": _num(leg.get("qty")),
+        "target_hrs": _num(leg.get("tgt", leg.get("target_hrs"))),
+        "op": clean_opt(leg.get("op"), LIMITS["op"]),
+        "dwg": clean_opt(leg.get("dwg"), LIMITS["drawing"]),
+        "rev": clean_opt(leg.get("rev"), LIMITS["rev"]),
+        "partspecs": clean_opt(leg.get("partspecs"), 4000),
+        "visual_status": clean_opt(leg.get("st", leg.get("visual_status")), 10),
+    }
+
+
+def _apply_legs(conn: Any, packet_id: int, legs: list[Any], title: str | None) -> dict[str, int]:
+    """Replace the packet's work orders with the change order's legs. Rows whose key and seq match are updated in
+    place (their ids and gone marks stay), new ones are added, the rest are deleted."""
+    want = [_leg_row(lg) for lg in legs]
+    have = packets.rows(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (packet_id,))
+
+    def k(r: dict[str, Any]) -> tuple[Any, ...]:
+        return (r["wo_base"], r["lot"], r["split"], r["sub"], r["seq"])
+
+    by_key = {k(r): r for r in have}
+    seen: set[tuple[Any, ...]] = set()
+    counts = {"added": 0, "updated": 0, "removed": 0}
+    for sort, r in enumerate(want):
+        key = k(r)
+        if key in seen:
+            continue
+        seen.add(key)
+        cur = by_key.get(key)
+        if cur is not None:
+            conn.execute(
+                "UPDATE packet_work_orders SET qty = %s, target_hrs = %s, op = %s, dwg = %s, rev = %s, partspecs = %s,"
+                " visual_status = %s, sort = %s, row_version = row_version + 1 WHERE id = %s",
+                (
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    r["visual_status"],
+                    sort,
+                    cur["id"],
+                ),
+            )
+            counts["updated"] += 1
+        else:
+            conn.execute(
+                "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op,"
+                " description, visual_status, dwg, rev, partspecs, sort)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    packet_id,
+                    r["wo_base"],
+                    r["lot"],
+                    r["split"],
+                    r["sub"],
+                    r["seq"],
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    title,
+                    r["visual_status"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    sort,
+                ),
+            )
+            counts["added"] += 1
+    gone = [r["id"] for r in have if k(r) not in seen]
+    if gone:
+        conn.execute("DELETE FROM packet_work_orders WHERE id = ANY(%s)", (gone,))
+        counts["removed"] = len(gone)
+    return counts
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+def _apply_feed_co(w: Write, co: dict[str, Any]) -> str | None:
+    """Apply a VISUAL or M-Files change order on approval: work orders, BOM, header drawing and rev, the fingerprint,
+    and the staged drawing (old pages voided, new pages carry the change order). Returns a note for the log."""
+    from . import packet_visual as pv
+
+    conn = w.conn
+    pid = w.id
+    vis = co["vis"] if isinstance(co["vis"], dict) else {}
+    staged = co["staged"] if isinstance(co["staged"], dict) else None
+    row = w.packet
+    visual = dict(row["visual"] or {})
+    notes: list[str] = []
+    sets: dict[str, Any] = {}
+    if isinstance(vis.get("legs"), list) and vis["legs"]:
+        c = _apply_legs(conn, pid, vis["legs"], row["title"])
+        notes.append(f"work orders: {c['added']} added, {c['updated']} kept, {c['removed']} removed")
+    if isinstance(vis.get("bom"), list) and not visual.get("manual_bom"):
+        c = pv.merge_visual_bom(conn, pid, vis["bom"])
+        notes.append(f"BOM: {c['added']} added, {c['updated']} kept, {c['removed'] + c['na']} dropped")
+    if co["origin"] == "visual":
+        if vis.get("to_dwg") not in (None, ""):
+            sets["drawing"] = clean_opt(vis["to_dwg"], LIMITS["drawing"])
+        if vis.get("to_rev") not in (None, ""):
+            sets["rev"] = clean_opt(vis["to_rev"], LIMITS["rev"])
+        if co["target_fp"]:
+            sets["visual_fp_applied"] = co["target_fp"]
+        for col, key in (("visual_want", "want"), ("visual_rel", "rel")):
+            if key in vis:
+                sets[col] = vis[key] or None
+        for key in ("fp_parts", "folded", "partspecs"):
+            if key in vis:
+                visual[key] = vis[key]
+    flags = list(row["flags"] or [])
+    if staged and staged.get("asset_id"):
+        asset = pv.pdf_asset(conn, staged["asset_id"])
+        dwg = sets.get("drawing") or staged.get("dwg") or row["drawing"]
+        rev = str(staged["rev"]) if staged.get("rev") not in (None, "") else (sets.get("rev") or row["rev"])
+        first, count, old = pv.replace_drawing_pages(
+            conn, pid, asset, dwg=dwg, rev=rev, co_id=int(co["id"]), void_by=w.user_id
+        )
+        for i in range(max(len(old), count)):
+            o = old[i] if i < len(old) else None
+            new = i < count
+            conn.execute(
+                "INSERT INTO co_items (co_id, old_page, old_dwg, old_rev, new_page, new_dwg, new_rev, added)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    co["id"],
+                    o["n"] if o else None,
+                    o["dwg"] if o else None,
+                    o["rev"] if o else None,
+                    first + i if new else None,
+                    dwg if new else None,
+                    rev if new else None,
+                    o is None,
+                ),
+            )
+        sets["drawing_state"] = "attached"
+        flags = pv.flags_without(flags, (*packets.DRAWING_FLAGS, "hand_drawing"))
+        visual["rev_confirmed"] = True
+        visual["vault"] = {
+            "asset_id": str(asset["id"]),
+            "name": staged.get("name"),
+            "rev": rev,
+            "at": pv.now_iso(),
+            "co_no": co["co_no"],
+        }
+        visual.pop("manual_dwg", None)
+        notes.append(f"drawing {dwg} rev {rev}, {_plural(count, 'page')}; {_plural(len(old), 'old page')} voided")
+    sets["flags"] = flags
+    sets["visual"] = Jsonb(visual)
+    cols = ", ".join(f"{c} = %s" + ("::drawing_state" if c == "drawing_state" else "") for c in sets)
+    conn.execute(f"UPDATE packets SET {cols} WHERE id = %s", (*sets.values(), pid))  # noqa: S608 (fixed names)
+    w.changed(*(c for c in sets if c not in ("flags", "visual")), "change_orders")
+    return "; ".join(notes) or None
+
+
@router.post("/{packet_id}/change-orders/{no}/approve", dependencies=WRITE)
 def approve_co(
    packet_id: int,
@@ -1401,9 +1586,19 @@ def approve_co(
    p: Principal = Depends(current_principal),  # noqa: B008
    d: Db = Depends(get_db),  # noqa: B008
 ) -> dict[str, Any]:
+    """Approve a pending change order. A VISUAL or M-Files one is applied to the packet here; a minor one (target
+    hours or want date only) needs no department verification."""
    pp = _pp(d, p)
    conn = d.conn
-    content = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    pre = packets.one(
+        conn,
+        "SELECT origin::text AS origin, staged, target_fp FROM change_orders WHERE packet_id = %s AND co_no = %s",
+        (packet_id, no),
+    )
+    content: dict[str, Any] = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    if pre is not None and pre["origin"] in FEED_ORIGINS:
+        content["target_fp"] = pre["target_fp"]
+        content["staged_sha256"] = pre["staged"].get("sha256") if isinstance(pre["staged"], dict) else None
    with packet_write(
        d,
        p,
@@ -1417,6 +1612,12 @@ def approve_co(
    ) as w:
        co = _co(conn, packet_id, no)
        _co_state(co, "pending", "approved")
+        note: str | None = None
+        if co["origin"] in FEED_ORIGINS:
+            vis = co["vis"] if isinstance(co["vis"], dict) else {}
+            if vis.get("dwg_change") is True and not co["staged"]:
+                raise ApiError(409, "co_not_staged", MSG_CO_NOT_STAGED, extra={"waiting": co["waiting"]})
+            note = _apply_feed_co(w, co)
        update_row(
            conn,
            "change_orders",
@@ -1433,7 +1634,7 @@ def approve_co(
            for code in co_touched(conn, w.packet):
                conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, %s)", (co["id"], code))
        _co_log_detail(w, no)
-        w.log(f"Change order {no} approved")
+        w.log(f"Change order {no} approved", note)
    return _read(d, pp, packet_id)