Phase 3: VISUAL feed and M-Files drawings in the worker

Migration 0004 (VISUAL and drawing caches, go-live ignored list, resource
map, job tables, order-derived columns with provenance; work order key
includes the operation sequence). The real worker on APScheduler with
settings re-read each fire, pause, run now, advisory locks, job_runs and a
heartbeat. The visual-feed job (five VisualKit reads, COPY and MERGE cache,
suspect-batch guard, the apply in one transaction: drafts, change orders,
retire-moved, work order gone, closed and cancelled observed, projects and
units from order links, auto-release) and the fingerprint. The drawing-lookup
job (queue, budget, download on version change, sha256 into assets, decision
table, staging for rev approval, drawing-origin change orders). Clerical From
VISUAL panel, flags, rev approval, hand uploads, change-order review; Admin
Feed settings and resource map. Synthetic fixture generator and a
fixture-driven worker in CI with a Playwright scenario. Docs updated.

Verified: 3249 server tests and 13 Playwright scenarios on the docker-1 CI
stack, 76 frontend tests, doctor at 0004; one feed and one drawing run
against VISUAL DEV and the vault from the developer machine (counts in the
changelog).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

server/packetstation/api/feed.py

@@ -0,0 +1,308 @@
+"""The VISUAL feed for Clerical: the From VISUAL panel (status, runs, ignored work orders, unmapped resources,
+projects without names), Run now, pulling in a work order released before go-live, and what VISUAL changed.
+
+Clerical (clerical or clerical_supervisor) or an admin. Pulling in a work order changes what the feed tracks, so an
+admin outside Clerical gives an IT reason for it. Screens read PostgreSQL only: nothing here calls VISUAL. Run now
+inserts a job request and NOTIFYs the worker, which does the reading.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..domain import packets
+from ..domain.pipeline import jsonable
+from ..text import LIMITS, clean_opt, reason_ok
+from .deps import Db, Principal, current_principal, get_db, require_password_session
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/feed", tags=["feed"])
+
+CLERICAL = frozenset({"clerical", "clerical_supervisor"})
+JOBS = ("visual-feed", "drawing-lookup")
+IGNORED_MAX = 200
+UNMAPPED_MAX = 30
+STALE_DEFAULT = 90
+PULL_IN_REASON_MIN = 3
+PULL_IN_REASON_MAX = 300
+
+
+class RunBody(BaseModel):
+    job: Literal["visual-feed", "drawing-lookup"] = "visual-feed"
+    note: str | None = Field(default=None, max_length=1000)
+
+
+class PullInBody(BaseModel):
+    reason: str | None = Field(default=None, max_length=2000)
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+# ---------------------------------------------------------------- access
+
+
+def require_clerical(p: Principal = Depends(current_principal)) -> Principal:  # noqa: B008
+    if CLERICAL & set(p.roles) or p.is_admin:
+        return p
+    raise ApiError(403, "forbidden", "Only Clerical can see the VISUAL feed.")
+
+
+def _it_reason(p: Principal, it_reason: str | None) -> str | None:
+    """None for Clerical. An admin outside Clerical needs a password session and an IT reason."""
+    if CLERICAL & set(p.roles):
+        return None
+    require_password_session(p)
+    why = reason_ok(it_reason)
+    if why is None:
+        raise ApiError(400, "it_reason_required", "Give an IT reason of at least 8 characters.")
+    return why
+
+
+# ---------------------------------------------------------------- shapes
+
+
+def _run_out(r: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": r["id"],
+        "job": r["job"],
+        "trigger": r["trigger"],
+        "requested_by": [str(u) for u in r.get("requested_by") or []],
+        "started_at": r["started_at"],
+        "finished_at": r["finished_at"],
+        "status": r["status"],
+        "counts": r["counts"] or {},
+        "errors": r["errors"] or [],
+        "build_version": r.get("build_version"),
+        "visual_target": r.get("visual_target"),
+        "host": r.get("host"),
+    }
+
+
+def _runs(conn: Any, job: str, limit: int) -> list[dict[str, Any]]:
+    return [
+        _run_out(r)
+        for r in packets.rows(conn, "SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit))
+    ]
+
+
+def _ignored_out(r: dict[str, Any]) -> dict[str, Any]:
+    keys = ("id", "wo_base", "lot", "split", "sub", "rel", "project", "unit_number", "res", "dwg", "last_seen_at")
+    out = {k: r[k] for k in keys}
+    out["pulled_in_at"] = r["pulled_in_at"]
+    out["pulled_in_by"] = str(r["pulled_in_by"]) if r["pulled_in_by"] else None
+    out["pulled_in_reason"] = r["pulled_in_reason"]
+    return out
+
+
+def _stale_seconds() -> int:
+    try:
+        from ..worker import STALE_SECONDS
+    except ImportError:  # the worker package is agent A's; the panel still answers without it
+        return STALE_DEFAULT
+    return int(STALE_SECONDS)
+
+
+def _feed_settings(conn: Any) -> Any:
+    from ..feed import config
+
+    return config.load(conn)
+
+
+def _request_run(conn: Any, p: Principal, job: str, note: str | None) -> int:
+    """Insert a job request and NOTIFY the worker in the same transaction. Returns the pending request count."""
+    conn.execute(
+        "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
+        (job, p.user_id, clean_opt(note, LIMITS["job_note"])),
+    )
+    conn.execute("SELECT ps_notify('ps_job_request', %s)", (json.dumps({"job": job}),))
+    return int(
+        conn.execute("SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)).fetchone()[0]
+    )
+
+
+# ---------------------------------------------------------------- endpoints
+
+
+@router.get("/overview")
+def overview(
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Everything the From VISUAL panel shows, from PostgreSQL only."""
+    conn = d.conn
+    fs = _feed_settings(conn)
+    sched = packets.one(conn, "SELECT next_run_at, paused FROM job_schedule WHERE job = 'visual-feed'")
+    beat = packets.one(
+        conn,
+        "SELECT at, build, running, now() - at > make_interval(secs => %s) AS stale FROM worker_heartbeat WHERE id = 1",
+        (_stale_seconds(),),
+    )
+    runs = _runs(conn, "visual-feed", 10)
+    last_run = runs[0] if runs else None
+    finished = next((r for r in runs if r["status"] in ("ok", "degraded")), None)
+    unmapped_raw = (finished or {}).get("counts", {}).get("unmapped") or {}
+    unmapped: list[dict[str, Any]] = []
+    if isinstance(unmapped_raw, dict):
+        for k, v in unmapped_raw.items():
+            if isinstance(v, int) or (isinstance(v, str) and v.isdigit()):
+                unmapped.append({"resource": str(k), "rows": int(v)})
+        unmapped.sort(key=lambda x: (-int(x["rows"]), str(x["resource"])))
+        unmapped = unmapped[:UNMAPPED_MAX]
+    ignored = packets.rows(
+        conn,
+        "SELECT * FROM visual_ignored_wos WHERE pulled_in_at IS NULL ORDER BY rel DESC NULLS LAST, wo_base, lot, id"
+        " LIMIT %s",
+        (IGNORED_MAX,),
+    )
+    nameless = packets.rows(
+        conn,
+        "SELECT p.id, p.number,"
+        " (SELECT count(*) FROM units u WHERE u.project_id = p.id AND u.removed_at IS NULL) AS units,"
+        " (SELECT count(*) FROM packets k JOIN units u ON u.id = k.unit_id"
+        "   WHERE u.project_id = p.id AND k.removed_at IS NULL) AS packets"
+        " FROM projects p WHERE p.source = 'visual' AND p.needs_details AND p.name IS NULL AND p.removed_at IS NULL"
+        " ORDER BY p.number LIMIT 200",
+    )
+    return {
+        "status": {
+            "paused": bool(fs.paused),
+            "next_run_at": (sched or {}).get("next_run_at"),
+            "worker": None
+            if beat is None
+            else {
+                "at": beat["at"],
+                "build": beat["build"],
+                "running": list(beat["running"] or []),
+                "stale": bool(beat["stale"]),
+            },
+        },
+        "last_run": last_run,
+        "runs": runs,
+        "drawing_runs": _runs(conn, "drawing-lookup", 10),
+        "pending_requests": int(
+            conn.execute(
+                "SELECT count(*) FROM job_requests WHERE job = 'visual-feed' AND claimed_at IS NULL"
+            ).fetchone()[0]
+        ),
+        "ignored": [_ignored_out(r) for r in ignored],
+        "unmapped": unmapped,
+        "projects_without_names": [
+            {"id": r["id"], "number": r["number"], "units": int(r["units"]), "packets": int(r["packets"])}
+            for r in nameless
+        ],
+        "golive_date": fs.golive_date,
+        "departments": list(fs.departments),
+    }
+
+
+@router.post("/run", dependencies=WRITE)
+def run_now(
+    body: RunBody | None = None,
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Run now: ask the worker for a feed (or drawing) run. Requests made while one is pending share that run."""
+    body = body or RunBody()
+    conn = d.conn
+    with conn.transaction():
+        pending = _request_run(conn, p, body.job, body.note)
+        audit.append(
+            conn,
+            action="feed.run_requested",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="job",
+            subject_id=body.job,
+            detail={"job": body.job, "pending": pending},
+            ip=p.ip,
+        )
+    return {"requested": True, "job": body.job, "pending": pending}
+
+
+@router.post("/ignored/{row_id}/pull-in", dependencies=WRITE)
+def pull_in(
+    row_id: int,
+    body: PullInBody,
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Track a work order released before the go-live date after all. The next feed run makes its packet."""
+    it_reason = _it_reason(p, body.it_reason)
+    reason = clean_opt(body.reason, PULL_IN_REASON_MAX)
+    if reason is None or len(reason) < PULL_IN_REASON_MIN:
+        raise ApiError(400, "reason_required", "Say why this work order is pulled in (at least 3 characters).")
+    conn = d.conn
+    with conn.transaction():
+        row = packets.one(conn, "SELECT * FROM visual_ignored_wos WHERE id = %s FOR UPDATE", (row_id,))
+        if row is None:
+            raise ApiError(404, "not_found", "No such work order on the ignored list.")
+        if row["pulled_in_at"] is not None:
+            raise ApiError(
+                409,
+                "already_pulled_in",
+                "This work order was already pulled in.",
+                extra={"current": jsonable(_ignored_out(row))},
+            )
+        row = packets.one(
+            conn,
+            "UPDATE visual_ignored_wos SET pulled_in_at = now(), pulled_in_by = %s, pulled_in_reason = %s"
+            " WHERE id = %s RETURNING *",
+            (p.user_id, reason, row_id),
+        )
+        assert row is not None
+        key = "/".join(str(row[k]) for k in ("wo_base", "lot", "split", "sub"))
+        audit.append(
+            conn,
+            action="feed.pulled_in",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="work_order",
+            subject_id=key,
+            detail={"id": row_id, "key": key, "rel": str(row["rel"]) if row["rel"] else None, "reason": reason},
+            it_reason=it_reason,
+            ip=p.ip,
+        )
+        _request_run(conn, p, "visual-feed", f"pull in {key}")
+    return _ignored_out(row)
+
+
+@router.get("/batches")
+def batches(
+    limit: int = Query(20, ge=1, le=200),
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> list[dict[str, Any]]:
+    return packets.rows(d.conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT %s", (limit,))
+
+
+@router.get("/changes")
+def changes(
+    batch_id: int | None = Query(None),
+    limit: int = Query(200, ge=1, le=2000),
+    p: Principal = Depends(require_clerical),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """What VISUAL changed in one batch (the latest when none is given): work order rows and material lines."""
+    conn = d.conn
+    if batch_id is None:
+        batch = packets.one(conn, "SELECT * FROM visual_batches ORDER BY id DESC LIMIT 1")
+    else:
+        batch = packets.one(conn, "SELECT * FROM visual_batches WHERE id = %s", (batch_id,))
+        if batch is None:
+            raise ApiError(404, "not_found", "No such feed batch.")
+    if batch is None:
+        return {"batch": None, "rows": [], "materials": []}
+    rows = packets.rows(
+        conn, "SELECT * FROM visual_row_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
+    )
+    mats = packets.rows(
+        conn, "SELECT * FROM visual_material_changes WHERE batch_id = %s ORDER BY id LIMIT %s", (batch["id"], limit)
+    )
+    return {"batch": batch, "rows": rows, "materials": mats}

server/packetstation/api/feed_settings.py

@@ -0,0 +1,204 @@
+"""Admin: the feed settings and the VISUAL resource map. Every route needs the admin role and a password session.
+
+A settings save NOTIFYs `ps_settings` so the worker reschedules at once. Resource map changes apply on the next
+feed run. Every change is audited with before and after.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, Body, Depends, Response
+from psycopg.rows import dict_row
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..domain.projects import Who
+from ..feed import config
+from .deps import Db, Principal, admin_principal, get_db
+from .errors import ApiError
+from .jobs import recent_runs
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/admin/feed", tags=["admin"])
+
+FEED_JOB = "visual-feed"
+SPECIAL_TARGETS = ("fold", "ignore")
+RESOURCE_MAX = 100
+
+
+def _who(p: Principal) -> Who:
+    return Who(user_id=p.user_id, session_id=p.session_id, role_snapshot=p.role_snapshot, ip=p.ip)
+
+
+def _shop_codes(conn: Any) -> set[str]:
+    rows = conn.execute("SELECT code FROM departments WHERE kind = 'shop' AND active").fetchall()
+    return {r[0] for r in rows}
+
+
+def _settings_out(conn: Any) -> dict[str, Any]:
+    fs, version, updated_at, updated_by = config.load_versioned(conn)
+    nxt = conn.execute("SELECT next_run_at FROM job_schedule WHERE job = %s", (FEED_JOB,)).fetchone()
+    return {
+        **fs.model_dump(mode="json"),
+        "version": version,
+        "updated_at": updated_at,
+        "updated_by": str(updated_by) if updated_by else None,
+        "next_run_at": nxt[0] if nxt else None,
+        "runs": recent_runs(conn, FEED_JOB, 10),
+    }
+
+
+@router.get("/settings")
+def get_settings(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    return _settings_out(d.conn)
+
+
+@router.patch("/settings", dependencies=WRITE)
+def patch_settings(
+    patch: dict[str, Any] = Body(...),  # noqa: B008
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    if "departments" in patch and isinstance(patch["departments"], list):
+        shop = _shop_codes(conn)
+        bad = sorted({str(x).strip() for x in patch["departments"]} - shop)
+        if bad:
+            raise ApiError(
+                422,
+                "bad_settings",
+                f"Not an active shop department: {', '.join(bad)}.",
+                extra={"errors": [f"departments: {b} is not an active shop department" for b in bad]},
+            )
+    try:
+        config.save(conn, who=_who(p), patch=patch)
+    except config.SettingsError as e:
+        raise ApiError(422, "bad_settings", str(e), extra={"errors": e.errors}) from None
+    return _settings_out(conn)
+
+
+# ---------------------------------------------------------------- resource map
+
+
+class MapBody(BaseModel):
+    target: str = Field(max_length=20)
+    confirmed: bool = True
+
+
+def _resource(resource: str) -> str:
+    r = resource.strip()
+    if not r or len(r) > RESOURCE_MAX:
+        raise ApiError(400, "bad_resource", "Give the VISUAL resource name.")
+    return r
+
+
+def _map_row(conn: Any, resource: str) -> dict[str, Any] | None:
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT resource, target, confirmed, updated_at, updated_by FROM visual_resource_map WHERE resource = %s",
+            (resource,),
+        ).fetchone()
+    return dict(r) if r else None
+
+
+def _row_out(r: dict[str, Any], rows_last_run: int | None) -> dict[str, Any]:
+    return {
+        "resource": r["resource"],
+        "target": r.get("target"),
+        "confirmed": bool(r.get("confirmed")),
+        "rows_last_run": rows_last_run,
+        "updated_at": r.get("updated_at"),
+    }
+
+
+def _last_counts(conn: Any) -> dict[str, int]:
+    rows = conn.execute(
+        "SELECT res, count(*) FROM visual_rows"
+        " WHERE last_batch = (SELECT max(id) FROM visual_batches) AND res <> '' GROUP BY res"
+    ).fetchall()
+    return {r[0]: int(r[1]) for r in rows}
+
+
+@router.get("/resource-map")
+def resource_map(p: Principal = Depends(admin_principal), d: Db = Depends(get_db)) -> list[dict[str, Any]]:  # noqa: B008
+    conn = d.conn
+    counts = _last_counts(conn)
+    with conn.cursor(row_factory=dict_row) as cur:
+        mapped = cur.execute("SELECT resource, target, confirmed, updated_at FROM visual_resource_map").fetchall()
+    out = {r["resource"]: _row_out(r, counts.get(r["resource"])) for r in mapped}
+    for res, n in counts.items():
+        if res not in out:
+            out[res] = {"resource": res, "target": None, "confirmed": False, "rows_last_run": n, "updated_at": None}
+    return sorted(out.values(), key=lambda r: (r["target"] is not None, r["resource"].lower()))
+
+
+def _public(r: dict[str, Any] | None) -> dict[str, Any] | None:
+    if r is None:
+        return None
+    return {"target": r["target"], "confirmed": r["confirmed"]}
+
+
+@router.put("/resource-map/{resource}", dependencies=WRITE)
+def map_resource(
+    resource: str,
+    body: MapBody,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    conn = d.conn
+    res = _resource(resource)
+    target = body.target.strip()
+    if target.lower() in SPECIAL_TARGETS:
+        target = target.lower()
+    elif target not in _shop_codes(conn):
+        raise ApiError(422, "bad_target", "The target is Fold, Ignore or an active shop department.")
+    with conn.transaction():
+        before = _map_row(conn, res)
+        conn.execute(
+            "INSERT INTO visual_resource_map (resource, target, confirmed, updated_at, updated_by)"
+            " VALUES (%s, %s, %s, now(), %s) ON CONFLICT (resource) DO UPDATE SET target = EXCLUDED.target,"
+            " confirmed = EXCLUDED.confirmed, updated_at = now(), updated_by = EXCLUDED.updated_by",
+            (res, target, body.confirmed, p.user_id),
+        )
+        after = _map_row(conn, res)
+        audit.append(
+            conn,
+            action="feed.resource_mapped",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="visual_resource",
+            subject_id=res,
+            detail={"resource": res, "before": _public(before), "after": _public(after)},
+            ip=p.ip,
+        )
+    assert after is not None
+    return _row_out(after, _last_counts(conn).get(res))
+
+
+@router.delete("/resource-map/{resource}", status_code=204, dependencies=WRITE)
+def unmap_resource(
+    resource: str,
+    p: Principal = Depends(admin_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> Response:
+    conn = d.conn
+    res = _resource(resource)
+    with conn.transaction():
+        before = _map_row(conn, res)
+        if before is None:
+            raise ApiError(404, "not_found", "That resource is not mapped.")
+        conn.execute("DELETE FROM visual_resource_map WHERE resource = %s", (res,))
+        audit.append(
+            conn,
+            action="feed.resource_unmapped",
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="visual_resource",
+            subject_id=res,
+            detail={"resource": res, "before": _public(before), "after": None},
+            ip=p.ip,
+        )
+    return Response(status_code=204)

server/packetstation/api/jobs.py

@@ -0,0 +1,170 @@
+"""The worker's jobs: schedule, heartbeat, recent runs, and "run now".
+
+Admin (password session) or Clerical (clerical or clerical_supervisor). "Run now" inserts a job_requests row and
+NOTIFYs `ps_job_request` in the same transaction; the worker claims every pending request in one run.
+`run_out`, `worker_status` and `request_run` are shared with api/feed.py.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query
+from psycopg.rows import dict_row
+from pydantic import BaseModel, Field
+
+from .. import audit
+from ..worker.heartbeat import STALE_SECONDS
+from .deps import Db, Principal, current_principal, get_db, require_password_session
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/jobs", tags=["jobs"])
+
+CLERICAL = frozenset({"clerical", "clerical_supervisor"})
+JOB_CHANNEL = "ps_job_request"
+_JOB_NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,39}$")
+
+
+def jobs_principal(p: Principal = Depends(current_principal)) -> Principal:  # noqa: B008
+    """Clerical in any session, or an admin with a password session."""
+    if CLERICAL & set(p.roles):
+        return p
+    if p.is_admin:
+        return require_password_session(p)
+    raise ApiError(403, "forbidden", "Only Clerical or an admin can see and run the worker's jobs.")
+
+
+class RunBody(BaseModel):
+    note: str | None = Field(None, max_length=500)
+
+
+def run_out(r: dict[str, Any] | None) -> dict[str, Any] | None:
+    """One job_runs row as JSON."""
+    if r is None:
+        return None
+    return {
+        "id": r["id"],
+        "job": r["job"],
+        "trigger": r["trigger"],
+        "requested_by": [str(u) for u in (r.get("requested_by") or [])],
+        "started_at": r["started_at"],
+        "finished_at": r["finished_at"],
+        "status": r["status"],
+        "counts": r.get("counts") or {},
+        "errors": r.get("errors") or [],
+        "build_version": r.get("build_version"),
+        "visual_target": r.get("visual_target"),
+        "host": r.get("host"),
+    }
+
+
+def recent_runs(conn: Any, job: str, limit: int = 10) -> list[dict[str, Any]]:
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute("SELECT * FROM job_runs WHERE job = %s ORDER BY id DESC LIMIT %s", (job, limit)).fetchall()
+    return [o for o in (run_out(r) for r in rows) if o is not None]
+
+
+def worker_status(conn: Any) -> dict[str, Any]:
+    """{at, build, host, running, stale}: stale when the worker has not beaten for STALE_SECONDS or never did."""
+    with conn.cursor(row_factory=dict_row) as cur:
+        r = cur.execute(
+            "SELECT at, build, host, running, now() - at > make_interval(secs => %s) AS stale"
+            " FROM worker_heartbeat WHERE id = 1",
+            (STALE_SECONDS,),
+        ).fetchone()
+    if r is None:
+        return {"at": None, "build": None, "host": None, "running": [], "stale": True}
+    return {
+        "at": r["at"],
+        "build": r["build"],
+        "host": r["host"],
+        "running": list(r["running"] or []),
+        "stale": bool(r["stale"]),
+    }
+
+
+def known_job(conn: Any, job: str) -> bool:
+    if not _JOB_NAME.match(job):
+        return False
+    return conn.execute("SELECT 1 FROM job_schedule WHERE job = %s", (job,)).fetchone() is not None
+
+
+def request_run(
+    conn: Any, p: Principal, job: str, *, note: str | None = None, action: str = "job.run_requested"
+) -> dict[str, Any]:
+    """Insert a request and NOTIFY the worker in one transaction. Returns {requested, pending}."""
+    with conn.transaction():
+        conn.execute(
+            "INSERT INTO job_requests (job, requested_by, note) VALUES (%s, %s, %s)",
+            (job, p.user_id, (note or "").strip() or None),
+        )
+        conn.execute("SELECT ps_notify(%s, %s)", (JOB_CHANNEL, json.dumps({"job": job}, separators=(",", ":"))))
+        pending = conn.execute(
+            "SELECT count(*) FROM job_requests WHERE job = %s AND claimed_at IS NULL", (job,)
+        ).fetchone()[0]
+        audit.append(
+            conn,
+            action=action,
+            user_id=p.user_id,
+            session_id=p.session_id,
+            role_snapshot=p.role_snapshot,
+            subject_type="job",
+            subject_id=job,
+            detail={"job": job, "pending": int(pending)},
+            ip=p.ip,
+        )
+    return {"requested": True, "pending": int(pending)}
+
+
+@router.get("")
+def list_jobs(p: Principal = Depends(jobs_principal), d: Db = Depends(get_db)) -> dict[str, Any]:  # noqa: B008
+    conn = d.conn
+    with conn.cursor(row_factory=dict_row) as cur:
+        rows = cur.execute(
+            "SELECT s.job, s.next_run_at, s.interval_seconds, s.paused, s.updated_at,"
+            " (SELECT count(*) FROM job_requests q WHERE q.job = s.job AND q.claimed_at IS NULL) AS pending,"
+            " CASE WHEN r.id IS NULL THEN NULL ELSE to_jsonb(r) END AS last_run"
+            " FROM job_schedule s"
+            " LEFT JOIN LATERAL (SELECT * FROM job_runs j WHERE j.job = s.job ORDER BY j.id DESC LIMIT 1) r ON true"
+            " ORDER BY s.job"
+        ).fetchall()
+    jobs = [
+        {
+            "job": r["job"],
+            "next_run_at": r["next_run_at"],
+            "interval_seconds": r["interval_seconds"],
+            "paused": r["paused"],
+            "updated_at": r["updated_at"],
+            "pending": int(r["pending"]),
+            "last_run": run_out(r["last_run"]),
+        }
+        for r in rows
+    ]
+    return {"jobs": jobs, "heartbeat": worker_status(conn)}
+
+
+@router.get("/{job}/runs")
+def job_runs(
+    job: str,
+    p: Principal = Depends(jobs_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+    limit: int = Query(10, ge=1, le=100),
+) -> list[dict[str, Any]]:
+    if not known_job(d.conn, job):
+        raise ApiError(404, "not_found", "No such job.")
+    return recent_runs(d.conn, job, limit)
+
+
+@router.post("/{job}/run", dependencies=WRITE)
+def run_now(
+    job: str,
+    body: RunBody | None = None,
+    p: Principal = Depends(jobs_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    if not known_job(d.conn, job):
+        raise ApiError(404, "not_found", "No such job.")
+    return request_run(d.conn, p, job, note=body.note if body else None)

server/packetstation/api/packet_children.py

@@ -1392,6 +1392,191 @@ def issue_co(
     return _read(d, pp, packet_id)
 
 
+FEED_ORIGINS = ("visual", "drawing")
+MSG_CO_NOT_STAGED = "The new drawing has not been fetched from M-Files yet."
+
+
+def _vis_text(leg: Any, *names: str) -> str:
+    for n in names:
+        v = leg.get(n) if isinstance(leg, dict) else None
+        if v is not None and v != "":
+            return str(v)
+    return ""
+
+
+def _leg_row(leg: Any) -> dict[str, Any]:
+    """One work order row from a change order's `vis.legs` entry. Accepts a flat leg ({wo|wo_base|base, lot, split,
+    sub, seq, ...}) or one with a nested key ({key: {base, lot, split, sub}, seq, ...})."""
+    if not isinstance(leg, dict):
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    key = leg.get("key") if isinstance(leg.get("key"), dict) else leg
+    base = _vis_text(key, "base", "wo_base", "wo")
+    if not base:
+        raise ApiError(422, "bad_change_order", "The change order's work order list cannot be read.")
+    seq_raw = leg.get("seq")
+    return {
+        "wo_base": base,
+        "lot": _vis_text(key, "lot"),
+        "split": _vis_text(key, "split"),
+        "sub": _vis_text(key, "sub"),
+        "seq": int(seq_raw) if seq_raw not in (None, "") else None,
+        "qty": _num(leg.get("qty")),
+        "target_hrs": _num(leg.get("tgt", leg.get("target_hrs"))),
+        "op": clean_opt(leg.get("op"), LIMITS["op"]),
+        "dwg": clean_opt(leg.get("dwg"), LIMITS["drawing"]),
+        "rev": clean_opt(leg.get("rev"), LIMITS["rev"]),
+        "partspecs": clean_opt(leg.get("partspecs"), 4000),
+        "visual_status": clean_opt(leg.get("st", leg.get("visual_status")), 10),
+    }
+
+
+def _apply_legs(conn: Any, packet_id: int, legs: list[Any], title: str | None) -> dict[str, int]:
+    """Replace the packet's work orders with the change order's legs. Rows whose key and seq match are updated in
+    place (their ids and gone marks stay), new ones are added, the rest are deleted."""
+    want = [_leg_row(lg) for lg in legs]
+    have = packets.rows(conn, "SELECT * FROM packet_work_orders WHERE packet_id = %s", (packet_id,))
+
+    def k(r: dict[str, Any]) -> tuple[Any, ...]:
+        return (r["wo_base"], r["lot"], r["split"], r["sub"], r["seq"])
+
+    by_key = {k(r): r for r in have}
+    seen: set[tuple[Any, ...]] = set()
+    counts = {"added": 0, "updated": 0, "removed": 0}
+    for sort, r in enumerate(want):
+        key = k(r)
+        if key in seen:
+            continue
+        seen.add(key)
+        cur = by_key.get(key)
+        if cur is not None:
+            conn.execute(
+                "UPDATE packet_work_orders SET qty = %s, target_hrs = %s, op = %s, dwg = %s, rev = %s, partspecs = %s,"
+                " visual_status = %s, sort = %s, row_version = row_version + 1 WHERE id = %s",
+                (
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    r["visual_status"],
+                    sort,
+                    cur["id"],
+                ),
+            )
+            counts["updated"] += 1
+        else:
+            conn.execute(
+                "INSERT INTO packet_work_orders (packet_id, wo_base, lot, split, sub, seq, qty, target_hrs, op,"
+                " description, visual_status, dwg, rev, partspecs, sort)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    packet_id,
+                    r["wo_base"],
+                    r["lot"],
+                    r["split"],
+                    r["sub"],
+                    r["seq"],
+                    r["qty"],
+                    r["target_hrs"],
+                    r["op"],
+                    title,
+                    r["visual_status"],
+                    r["dwg"],
+                    r["rev"],
+                    r["partspecs"],
+                    sort,
+                ),
+            )
+            counts["added"] += 1
+    gone = [r["id"] for r in have if k(r) not in seen]
+    if gone:
+        conn.execute("DELETE FROM packet_work_orders WHERE id = ANY(%s)", (gone,))
+        counts["removed"] = len(gone)
+    return counts
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+def _apply_feed_co(w: Write, co: dict[str, Any]) -> str | None:
+    """Apply a VISUAL or M-Files change order on approval: work orders, BOM, header drawing and rev, the fingerprint,
+    and the staged drawing (old pages voided, new pages carry the change order). Returns a note for the log."""
+    from . import packet_visual as pv
+
+    conn = w.conn
+    pid = w.id
+    vis = co["vis"] if isinstance(co["vis"], dict) else {}
+    staged = co["staged"] if isinstance(co["staged"], dict) else None
+    row = w.packet
+    visual = dict(row["visual"] or {})
+    notes: list[str] = []
+    sets: dict[str, Any] = {}
+    if isinstance(vis.get("legs"), list) and vis["legs"]:
+        c = _apply_legs(conn, pid, vis["legs"], row["title"])
+        notes.append(f"work orders: {c['added']} added, {c['updated']} kept, {c['removed']} removed")
+    if isinstance(vis.get("bom"), list) and not visual.get("manual_bom"):
+        c = pv.merge_visual_bom(conn, pid, vis["bom"])
+        notes.append(f"BOM: {c['added']} added, {c['updated']} kept, {c['removed'] + c['na']} dropped")
+    if co["origin"] == "visual":
+        if vis.get("to_dwg") not in (None, ""):
+            sets["drawing"] = clean_opt(vis["to_dwg"], LIMITS["drawing"])
+        if vis.get("to_rev") not in (None, ""):
+            sets["rev"] = clean_opt(vis["to_rev"], LIMITS["rev"])
+        if co["target_fp"]:
+            sets["visual_fp_applied"] = co["target_fp"]
+        for col, key in (("visual_want", "want"), ("visual_rel", "rel")):
+            if key in vis:
+                sets[col] = vis[key] or None
+        for key in ("fp_parts", "folded", "partspecs"):
+            if key in vis:
+                visual[key] = vis[key]
+    flags = list(row["flags"] or [])
+    if staged and staged.get("asset_id"):
+        asset = pv.pdf_asset(conn, staged["asset_id"])
+        dwg = sets.get("drawing") or staged.get("dwg") or row["drawing"]
+        rev = str(staged["rev"]) if staged.get("rev") not in (None, "") else (sets.get("rev") or row["rev"])
+        first, count, old = pv.replace_drawing_pages(
+            conn, pid, asset, dwg=dwg, rev=rev, co_id=int(co["id"]), void_by=w.user_id
+        )
+        for i in range(max(len(old), count)):
+            o = old[i] if i < len(old) else None
+            new = i < count
+            conn.execute(
+                "INSERT INTO co_items (co_id, old_page, old_dwg, old_rev, new_page, new_dwg, new_rev, added)"
+                " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    co["id"],
+                    o["n"] if o else None,
+                    o["dwg"] if o else None,
+                    o["rev"] if o else None,
+                    first + i if new else None,
+                    dwg if new else None,
+                    rev if new else None,
+                    o is None,
+                ),
+            )
+        sets["drawing_state"] = "attached"
+        flags = pv.flags_without(flags, (*packets.DRAWING_FLAGS, "hand_drawing"))
+        visual["rev_confirmed"] = True
+        visual["vault"] = {
+            "asset_id": str(asset["id"]),
+            "name": staged.get("name"),
+            "rev": rev,
+            "at": pv.now_iso(),
+            "co_no": co["co_no"],
+        }
+        visual.pop("manual_dwg", None)
+        notes.append(f"drawing {dwg} rev {rev}, {_plural(count, 'page')}; {_plural(len(old), 'old page')} voided")
+    sets["flags"] = flags
+    sets["visual"] = Jsonb(visual)
+    cols = ", ".join(f"{c} = %s" + ("::drawing_state" if c == "drawing_state" else "") for c in sets)
+    conn.execute(f"UPDATE packets SET {cols} WHERE id = %s", (*sets.values(), pid))  # noqa: S608 (fixed names)
+    w.changed(*(c for c in sets if c not in ("flags", "visual")), "change_orders")
+    return "; ".join(notes) or None
+
+
 @router.post("/{packet_id}/change-orders/{no}/approve", dependencies=WRITE)
 def approve_co(
     packet_id: int,
@@ -1401,9 +1586,19 @@ def approve_co(
     p: Principal = Depends(current_principal),  # noqa: B008
     d: Db = Depends(get_db),  # noqa: B008
 ) -> dict[str, Any]:
+    """Approve a pending change order. A VISUAL or M-Files one is applied to the packet here; a minor one (target
+    hours or want date only) needs no department verification."""
     pp = _pp(d, p)
     conn = d.conn
-    content = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    pre = packets.one(
+        conn,
+        "SELECT origin::text AS origin, staged, target_fp FROM change_orders WHERE packet_id = %s AND co_no = %s",
+        (packet_id, no),
+    )
+    content: dict[str, Any] = {"action": str(A.CO_APPROVE), "packet_id": packet_id, "co_no": no}
+    if pre is not None and pre["origin"] in FEED_ORIGINS:
+        content["target_fp"] = pre["target_fp"]
+        content["staged_sha256"] = pre["staged"].get("sha256") if isinstance(pre["staged"], dict) else None
     with packet_write(
         d,
         p,
@@ -1417,6 +1612,12 @@ def approve_co(
     ) as w:
         co = _co(conn, packet_id, no)
         _co_state(co, "pending", "approved")
+        note: str | None = None
+        if co["origin"] in FEED_ORIGINS:
+            vis = co["vis"] if isinstance(co["vis"], dict) else {}
+            if vis.get("dwg_change") is True and not co["staged"]:
+                raise ApiError(409, "co_not_staged", MSG_CO_NOT_STAGED, extra={"waiting": co["waiting"]})
+            note = _apply_feed_co(w, co)
         update_row(
             conn,
             "change_orders",
@@ -1433,7 +1634,7 @@ def approve_co(
             for code in co_touched(conn, w.packet):
                 conn.execute("INSERT INTO co_required_depts (co_id, dept_code) VALUES (%s, %s)", (co["id"], code))
         _co_log_detail(w, no)
-        w.log(f"Change order {no} approved")
+        w.log(f"Change order {no} approved", note)
     return _read(d, pp, packet_id)
 
 

server/packetstation/api/packet_visual.py

@@ -0,0 +1,580 @@
+"""VISUAL packets by hand: approve the M-Files rev, upload the drawing or the BOM on a flagged draft, and go back to
+VISUAL's BOM.
+
+Every write goes through domain.pipeline.packet_write with its policy action and a signature. The hand uploads are
+for a VISUAL draft that is flagged (or already has a hand upload): Clerical checks it and sends it on; auto-release
+never takes it because its flags are not empty. Every answer is the full read model.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from psycopg.types.json import Jsonb
+from pydantic import BaseModel, Field
+from starlette.concurrency import run_in_threadpool
+from starlette.datastructures import UploadFile
+
+from .. import policy
+from ..domain import assets as asset_store
+from ..domain import handbom, packets
+from ..domain.pipeline import SignBody, Write, next_counter, packet_write
+from ..text import LIMITS, clean_opt, reason_ok
+from .deps import Db, Principal, current_principal, get_db
+from .errors import ApiError
+from .packets import WRITE
+
+router = APIRouter(prefix="/api/packets", tags=["packet-visual"])
+
+A = policy.Action
+
+# A hand BOM file is a small sheet. The multipart overhead on top is allowed before the body is refused unread.
+BOM_MAX_BYTES = 5 * 1024 * 1024
+FORM_SLACK = 64 * 1024
+
+MSG_NOT_FLAGGED = (
+    "A drawing or BOM is uploaded by hand only on a flagged VISUAL draft. This one is not flagged: send it on as it is."
+)
+MSG_NOT_VISUAL = "This is not a packet made from VISUAL."
+MSG_DRAFT_ONLY = "Only a draft takes a hand upload. Once it leaves Clerical, changes come in as change orders."
+
+
+# ---------------------------------------------------------------- bodies
+
+
+class RevApproveBody(BaseModel):
+    reason: str | None = Field(default=None, max_length=2000)
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+class HandDrawingBody(BaseModel):
+    asset_id: uuid.UUID
+    reason: str | None = Field(default=None, max_length=2000)
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+class HandBomOffBody(BaseModel):
+    sign: SignBody | None = None
+    it_reason: str | None = Field(default=None, max_length=1000)
+
+
+# ---------------------------------------------------------------- shared helpers
+
+
+def _pp(d: Db, p: Principal) -> policy.Principal:
+    return packets.policy_principal_for(d.conn, p)
+
+
+def _read(d: Db, pp: policy.Principal, packet_id: int) -> dict[str, Any]:
+    return packets.read_packet(d.conn, packet_id, pp)
+
+
+def now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def who_of(w: Write) -> dict[str, Any]:
+    return {"by": str(w.user_id), "by_name": w.principal.user.get("display_name")}
+
+
+def hand_flagged(row: Mapping[str, Any]) -> bool:
+    """The hand-upload guard: a VISUAL draft that is flagged, still needs its drawing, or already has a hand upload."""
+    if row["source_kind"] != "visual" or row["status"] != "draft":
+        return False
+    visual = row["visual"] or {}
+    return bool(
+        list(row["flags"] or [])
+        or row["drawing_state"] in packets.DRAWING_STATES_FLAGGED
+        or visual.get("manual_dwg")
+        or visual.get("manual_bom")
+    )
+
+
+def _hand_guard(w: Write) -> None:
+    row = w.packet
+    if row["source_kind"] != "visual":
+        raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+    if row["status"] != "draft":
+        raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
+    if not hand_flagged(row):
+        raise ApiError(409, "not_flagged", MSG_NOT_FLAGGED)
+
+
+def pdf_asset(conn: Any, asset_id: Any) -> dict[str, Any]:
+    asset = asset_store.get(conn, asset_id)
+    if asset is None:
+        raise ApiError(404, "not_found", "No such file.")
+    if asset["mime"] != asset_store.PDF or not asset["pages"]:
+        raise ApiError(422, "not_pdf", "Attach a PDF drawing.")
+    return asset
+
+
+def flags_without(flags: Any, drop: tuple[str, ...], add: tuple[str, ...] = ()) -> list[str]:
+    out = [f for f in (flags or []) if f not in drop]
+    for f in add:
+        if f not in out:
+            out.append(f)
+    return out
+
+
+def replace_drawing_pages(
+    conn: Any,
+    packet_id: int,
+    asset: Mapping[str, Any],
+    *,
+    dwg: str | None,
+    rev: str | None,
+    co_id: int | None = None,
+    void_by: Any = None,
+) -> tuple[int, int, list[dict[str, Any]]]:
+    """Put an asset's pages on the packet as its drawing.
+
+    Without `co_id` (a draft) the current drawing pages are deleted (the assets stay). With `co_id` (a change order
+    approval) they are voided and point at the first new page. Returns (first new page, page count, old pages).
+    """
+    old = packets.rows(
+        conn,
+        "SELECT * FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL ORDER BY n",
+        (packet_id,),
+    )
+    count = int(asset["pages"] or 0)
+    if count < 1:
+        raise ApiError(422, "not_pdf", "That drawing has no pages.")
+    first = next_counter(conn, packet_id, "page", count)
+    if co_id is None:
+        conn.execute(
+            "DELETE FROM packet_pages WHERE packet_id = %s AND type = 'drawing' AND void_at IS NULL", (packet_id,)
+        )
+    elif old:
+        conn.execute(
+            "UPDATE packet_pages SET void_co_id = %s, void_at = now(), void_by = %s, void_new_n = %s,"
+            " row_version = row_version + 1 WHERE id = ANY(%s)",
+            (co_id, void_by, first, [pg["id"] for pg in old]),
+        )
+    for i in range(count):
+        conn.execute(
+            "INSERT INTO packet_pages (packet_id, n, type, asset_id, pn, dwg, rev, co_id)"
+            " VALUES (%s, %s, 'drawing', %s, %s, %s, %s, %s)",
+            (packet_id, first + i, asset["id"], i + 1, dwg, rev, co_id),
+        )
+    return first, count, old
+
+
+def set_visual(conn: Any, packet_id: int, visual: Mapping[str, Any]) -> None:
+    conn.execute("UPDATE packets SET visual = %s WHERE id = %s", (Jsonb(dict(visual)), packet_id))
+
+
+def _sha(asset: Mapping[str, Any]) -> str:
+    return bytes(asset["sha256"]).hex()
+
+
+# ---------------------------------------------------------------- approve the M-Files rev
+
+
+@router.post("/{packet_id}/rev-approve", dependencies=WRITE)
+def rev_approve(
+    packet_id: int,
+    body: RevApproveBody,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """A Clerical Supervisor approves the rev M-Files has in place of the rev VISUAL calls, for this draft only."""
+    conn = d.conn
+    reason = reason_ok(body.reason)
+    pp = _pp(d, p)
+    row = packets.load_row(conn, packet_id)
+    staged = packets.staged_lookup(conn, row["drawing"]) if row else None
+    content = {
+        "action": str(A.REV_APPROVE),
+        "packet_id": packet_id,
+        "drawing": row["drawing"] if row else None,
+        "visual_rev": row["rev"] if row else None,
+        "rev": staged["rev"] if staged else None,
+        "sha256": bytes(staged["sha256"]).hex() if staged and staged["sha256"] else None,
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.REV_APPROVE,
+        request=request,
+        extra={"reason": body.reason},
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content=content,
+        policy_principal=pp,
+    ) as w:
+        if w.packet["source_kind"] != "visual":
+            raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+        lk = packets.staged_lookup(conn, w.packet["drawing"])
+        if lk is None:
+            raise ApiError(
+                409, "nothing_staged", "No M-Files copy of this drawing is ready yet. Try after the next run."
+            )
+        asset = asset_store.get(conn, lk["asset_id"])
+        if asset is None or not asset["pages"]:
+            raise ApiError(
+                409, "nothing_staged", "The M-Files copy of this drawing is missing. Try after the next run."
+            )
+        rev = str(lk["rev"])
+        vis_rev = w.packet["rev"] or ""
+        replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=rev)
+        visual = dict(w.packet["visual"] or {})
+        visual["rev_ok"] = {"rev": rev, "vis_rev": vis_rev, **who_of(w), "at": now_iso(), "reason": reason}
+        visual["rev_confirmed"] = True
+        visual["vault"] = {
+            "asset_id": str(asset["id"]),
+            "name": lk["name"],
+            "rev": rev,
+            "note": lk["note"],
+            "at": now_iso(),
+        }
+        visual.pop("manual_dwg", None)
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET drawing_state = 'attached', flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], (*packets.DRAWING_FLAGS, "hand_drawing")), packet_id),
+        )
+        w.changed("pages", "drawing_state", "flags", "visual")
+        w.detail.update(asset_id=str(asset["id"]), rev=rev, vis_rev=vis_rev)
+        w.log("Different rev approved by hand", f"rev {rev} in place of VISUAL rev {vis_rev or '-'} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- hand drawing
+
+
+@router.post("/{packet_id}/hand/drawing", dependencies=WRITE)
+def hand_drawing(
+    packet_id: int,
+    body: HandDrawingBody,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Clerical uploads the drawing by hand on a flagged VISUAL draft. It replaces the drawing pages."""
+    conn = d.conn
+    reason = clean_opt(body.reason, LIMITS["reason"])
+    if reason is None:
+        raise ApiError(400, "reason_required", "Say why the drawing is uploaded by hand.")
+    asset = pdf_asset(conn, body.asset_id)
+    pp = _pp(d, p)
+    content = {
+        "action": "hand_drawing",
+        "packet_id": packet_id,
+        "asset_id": str(asset["id"]),
+        "sha256": _sha(asset),
+        "pages": asset["pages"],
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_DOCUMENT_SET,
+        request=request,
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content=content,
+        require_sign=True,
+        policy_principal=pp,
+    ) as w:
+        _hand_guard(w)
+        vis_rev = w.packet["rev"] or ""
+        first, count, _ = replace_drawing_pages(conn, packet_id, asset, dwg=w.packet["drawing"], rev=w.packet["rev"])
+        name = asset["original_name"] or "drawing.pdf"
+        visual = dict(w.packet["visual"] or {})
+        visual["manual_dwg"] = {
+            "asset_id": str(asset["id"]),
+            "name": name,
+            "reason": reason,
+            **who_of(w),
+            "at": now_iso(),
+            "vis_rev": vis_rev,
+        }
+        visual["rev_confirmed"] = False
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET drawing_state = 'hand', flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], packets.DRAWING_FLAGS, ("hand_drawing",)), packet_id),
+        )
+        w.changed("pages", "drawing_state", "flags", "visual")
+        w.detail.update(asset_id=str(asset["id"]), pages=count, first_page=first)
+        w.log("Drawing uploaded by hand", f"{name}, {count} page{'s' if count != 1 else ''} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- hand BOM
+
+
+def _insert_hand_bom(conn: Any, packet_id: int, lines: list[dict[str, Any]]) -> None:
+    conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+    for i, b in enumerate(lines):
+        conn.execute(
+            "INSERT INTO bom_lines (packet_id, item, qty, part_id, description, dwg_ref, cl_item, length, material,"
+            " src, sort) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'hand', %s)",
+            (
+                packet_id,
+                b["item"],
+                b["qty"],
+                b["part_id"],
+                b["description"],
+                b["dwg_ref"],
+                b["cl_item"],
+                b["length"],
+                b["material"],
+                i,
+            ),
+        )
+
+
+def _hand_bom_write(
+    d: Db,
+    p: Principal,
+    request: Request,
+    packet_id: int,
+    *,
+    data: bytes,
+    name: str | None,
+    reason: str,
+    sign: SignBody,
+    it_reason: str | None,
+) -> dict[str, Any]:
+    conn = d.conn
+    try:
+        lines = handbom.parse(data, name)
+    except handbom.BadFile as e:
+        raise ApiError(422, "bad_file", str(e)) from None
+    pp = _pp(d, p)
+    sha = hashlib.sha256(data).hexdigest()
+    fname = clean_opt(name, LIMITS["file_name"]) or "bom"
+    content = {
+        "action": "hand_bom",
+        "packet_id": packet_id,
+        "file": fname,
+        "sha256": sha,
+        "lines": len(lines),
+        "reason": reason,
+    }
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_BOM,
+        request=request,
+        it_reason=it_reason,
+        sign=sign,
+        sign_content=content,
+        require_sign=True,
+        policy_principal=pp,
+    ) as w:
+        _hand_guard(w)
+        _insert_hand_bom(conn, packet_id, lines)
+        visual = dict(w.packet["visual"] or {})
+        visual["manual_bom"] = {
+            "name": fname,
+            "reason": reason,
+            **who_of(w),
+            "at": now_iso(),
+            "lines": len(lines),
+            "sha256": sha,
+        }
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], (), ("hand_bom",)), packet_id),
+        )
+        w.changed("bom", "flags", "visual")
+        w.detail.update(file=fname, sha256=sha, lines=len(lines))
+        n = len(lines)
+        w.log("BOM uploaded by hand", f"{fname} · {n} line{'s' if n != 1 else ''} · {reason}")
+    return _read(d, pp, packet_id)
+
+
+@router.post("/{packet_id}/hand/bom")
+async def hand_bom(
+    packet_id: int,
+    request: Request,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Clerical uploads the BOM by hand (multipart: file, reason, sign_factor, sign_secret, it_reason)."""
+    length = request.headers.get("content-length")
+    if length is not None and length.isdigit() and int(length) > BOM_MAX_BYTES + FORM_SLACK:
+        raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
+    form = await request.form(max_files=1, max_fields=8)
+    try:
+        file = form.get("file")
+        if not isinstance(file, UploadFile):
+            raise ApiError(422, "invalid_request", "Send the BOM file in the 'file' field.")
+
+        def field(name: str, cap: int) -> str | None:
+            v = form.get(name)
+            return v[:cap] if isinstance(v, str) else None
+
+        reason = clean_opt(field("reason", 2000), LIMITS["reason"])
+        if reason is None:
+            raise ApiError(400, "reason_required", "Say why the BOM is uploaded by hand.")
+        factor, secret = field("sign_factor", 20), field("sign_secret", 1024)
+        if not factor or secret is None:
+            raise ApiError(
+                400, "signature_required", "This step needs your signature.", extra={"meaning": "Edit the BOM"}
+            )
+        if factor not in ("pin", "password"):
+            raise ApiError(422, "invalid_request", "sign_factor is pin or password.")
+        sign = SignBody(factor=factor, secret=secret)  # type: ignore[arg-type]
+        data = await file.read(BOM_MAX_BYTES + 1)
+        if len(data) > BOM_MAX_BYTES:
+            raise ApiError(413, "too_large", f"The BOM file is over the {BOM_MAX_BYTES // (1024 * 1024)} MB limit.")
+        name = file.filename
+        it_reason = field("it_reason", 1000)
+    finally:
+        await form.close()
+    return await run_in_threadpool(
+        lambda: _hand_bom_write(
+            d, p, request, packet_id, data=data, name=name, reason=reason, sign=sign, it_reason=it_reason
+        )
+    )
+
+
+@router.delete("/{packet_id}/hand/bom", dependencies=WRITE)
+def hand_bom_off(
+    packet_id: int,
+    request: Request,
+    body: HandBomOffBody | None = None,
+    p: Principal = Depends(current_principal),  # noqa: B008
+    d: Db = Depends(get_db),  # noqa: B008
+) -> dict[str, Any]:
+    """Drop the hand BOM and use VISUAL's BOM again, rebuilt from the cached materials."""
+    body = body or HandBomOffBody()
+    conn = d.conn
+    pp = _pp(d, p)
+    with packet_write(
+        d,
+        p,
+        packet_id,
+        A.EDIT_BOM,
+        request=request,
+        it_reason=body.it_reason,
+        sign=body.sign,
+        sign_content={"action": "hand_bom_off", "packet_id": packet_id},
+        require_sign=body.sign is not None,
+        policy_principal=pp,
+    ) as w:
+        if w.packet["source_kind"] != "visual":
+            raise ApiError(409, "not_visual", MSG_NOT_VISUAL)
+        if w.packet["status"] != "draft":
+            raise ApiError(403, "wrong_status", MSG_DRAFT_ONLY)
+        visual = dict(w.packet["visual"] or {})
+        old = visual.pop("manual_bom", None)
+        if not old:
+            raise ApiError(409, "no_hand_bom", "This packet has no BOM uploaded by hand.")
+        from ..feed import materials as feed_materials  # agent B's module, imported where it is used
+
+        lines = feed_materials.bom_for_packet(conn, packet_id)
+        conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+        feed_materials.insert_bom(conn, packet_id, lines)
+        set_visual(conn, packet_id, visual)
+        conn.execute(
+            "UPDATE packets SET flags = %s WHERE id = %s",
+            (flags_without(w.packet["flags"], ("hand_bom",)), packet_id),
+        )
+        w.changed("bom", "flags", "visual")
+        n = len(lines)
+        name = old.get("name") if isinstance(old, dict) else None
+        note = f"{n} line{'s' if n != 1 else ''} from VISUAL" if n else "VISUAL has no materials for it yet"
+        w.log("Hand-uploaded BOM taken off", " · ".join(x for x in (name, note) if x))
+    return _read(d, pp, packet_id)
+
+
+# ---------------------------------------------------------------- VISUAL BOM lines (change order approval)
+
+
+def merge_visual_bom(conn: Any, packet_id: int, raw_lines: list[Any]) -> dict[str, int]:
+    """Bring a packet's BOM to VISUAL's lines (change order `vis.bom`, dicts or BomLine objects) without losing work.
+
+    When nothing has been recorded on the lines (no pulled mark, heat, MTR or short) the BOM is replaced through
+    feed.materials.insert_bom, exactly as the feed writes it. Otherwise lines are matched on vkey and updated in
+    place, so pulled marks, heats and shorts stay. New lines are added with the same encoding (item numbers from
+    feed.materials.db_items, the label in cl_item, cons Y/N, sub Y or NULL). Lines VISUAL dropped are deleted, or
+    marked n/a when they were pulled or have a short.
+    """
+    from ..feed.materials import BomLine, db_items, insert_bom
+
+    lines = [ln if isinstance(ln, BomLine) else BomLine.from_dict(ln) for ln in raw_lines]
+    counts = {"added": 0, "updated": 0, "removed": 0, "na": 0}
+    existing = packets.rows(conn, "SELECT * FROM bom_lines WHERE packet_id = %s ORDER BY item", (packet_id,))
+    worked = (
+        any(e["pulled"] or e["heat"] or e["mtr_asset_id"] for e in existing)
+        or conn.execute("SELECT 1 FROM bom_shorts WHERE packet_id = %s LIMIT 1", (packet_id,)).fetchone()
+    )
+    if not worked:
+        # Nothing recorded on the lines yet: write them exactly as the feed does.
+        conn.execute("DELETE FROM bom_lines WHERE packet_id = %s", (packet_id,))
+        insert_bom(conn, packet_id, lines)
+        counts["added"] = len(lines)
+        counts["removed"] = len(existing)
+        return counts
+    by_vkey = {e["vkey"]: e for e in existing if e["vkey"]}
+    used = {int(e["item"]) for e in existing}
+    wanted = db_items(lines)
+    kept: set[int] = set()
+    for sort, (b, want_item) in enumerate(zip(lines, wanted, strict=True)):
+        values = (
+            b.qty,
+            b.part_id[:60] or None,
+            b.desc[:240] or None,
+            b.loc[:120] or None,
+            b.whs or None,
+            b.comm or None,
+            "Y" if b.cons else "N",
+            "Y" if b.sub else None,
+            b.na,
+            b.grp,
+            b.issued,
+            sort,
+        )
+        match = by_vkey.get(b.vkey) if b.vkey else None
+        if match is not None and int(match["id"]) not in kept:
+            kept.add(int(match["id"]))
+            conn.execute(
+                "UPDATE bom_lines SET qty = %s, part_id = %s, description = %s, loc = %s, whs = %s, comm = %s,"
+                " cons = %s, sub = %s, na = %s, grp = %s, issued = coalesce(%s, issued), sort = %s, cl_item = %s,"
+                " row_version = row_version + 1 WHERE id = %s",
+                (*values, b.item, match["id"]),
+            )
+            counts["updated"] += 1
+            continue
+        item = want_item
+        while item in used:
+            item += 1
+        used.add(item)
+        new_id = conn.execute(
+            "INSERT INTO bom_lines (packet_id, item, cl_item, qty, part_id, description, loc, whs, comm, cons, sub, na,"
+            " grp, issued, sort, vkey, src) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,"
+            " 'visual') RETURNING id",
+            (packet_id, item, b.item, *values, b.vkey),
+        ).fetchone()[0]
+        kept.add(int(new_id))
+        counts["added"] += 1
+    for e in existing:
+        if int(e["id"]) in kept:
+            continue
+        busy = e["pulled"] or conn.execute("SELECT 1 FROM bom_shorts WHERE bom_line_id = %s", (e["id"],)).fetchone()
+        if busy:
+            if not e["na"]:
+                conn.execute("UPDATE bom_lines SET na = true, row_version = row_version + 1 WHERE id = %s", (e["id"],))
+                counts["na"] += 1
+        else:
+            conn.execute("DELETE FROM bom_lines WHERE id = %s", (e["id"],))
+            counts["removed"] += 1
+    return counts

server/packetstation/api/packets.py

@@ -299,6 +299,7 @@ def list_packets(
     project_id: int | None = Query(None),
     q: str = Query("", max_length=200),
     flag: str = Query("", max_length=20),
+    co: str = Query("", max_length=20),
     limit: int = Query(500, ge=1, le=2000),
     removed: int = Query(0, ge=0, le=1),
 ) -> dict[str, Any]:
@@ -309,6 +310,7 @@ def list_packets(
         "project_id": project_id,
         "q": clean_text(q, 200),
         "flag": flag,
+        "co": co.strip().lower() or None,
         "limit": limit,
         "removed": bool(removed) and p.is_admin,
     }